def test_check_fit_params(indices): X = np.random.randn(4, 2) fit_params = { 'list': [1, 2, 3, 4], 'array': np.array([1, 2, 3, 4]), 'sparse-col': sp.csc_matrix([1, 2, 3, 4]).T, 'sparse-row': sp.csc_matrix([1, 2, 3, 4]), 'scalar-int': 1, 'scalar-str': 'xxx', 'None': None, } result = _check_fit_params(X, fit_params, indices) indices_ = indices if indices is not None else list(range(X.shape[0])) for key in ['sparse-row', 'scalar-int', 'scalar-str', 'None']: assert result[key] is fit_params[key] assert result['list'] == _safe_indexing(fit_params['list'], indices_) assert_array_equal( result['array'], _safe_indexing(fit_params['array'], indices_) ) assert_allclose_dense_sparse( result['sparse-col'], _safe_indexing(fit_params['sparse-col'], indices_) )
def fit(self, X, Y, sample_weight=None, **fit_params): """Fit the model to data matrix X and targets Y. Parameters ---------- X : (sparse) array-like, shape (n_samples, n_features) input data. Y : (sparse) array-like, shape (n_samples, n_outputs) Multi-output targets. An indicator matrix turns on multilabel estimation. sample_weight : array-like of shape (n_samples,) or None Sample weights. If None, then samples are equally weighted. Only supported if the underlying classifier supports sample weights. **fit_params : dict of string -> object Parameters passed to the ``estimator.fit`` method of each step. Returns ------- self : object """ self._validate_estimators() for _, est in self.estimators: if not hasattr(est, 'fit'): raise AttributeError( 'Every base estimator should implement a fit method.') X, Y = self._validate_data(X, Y, multi_output=True, accept_sparse=True) if is_classifier(self): check_classification_targets(Y) if Y.ndim == 1: raise ValueError( 'Output Y must have at least two dimensions for multi-output classification but has only one.' ) if sample_weight is not None and any([ not has_fit_parameter(clf, 'sample_weight') for _, clf in self.estimators ]): raise ValueError( 'One of base estimators does not support sample weights.') fit_params_validated = _check_fit_params(X, fit_params) self.estimators_ = Parallel(n_jobs=self.n_jobs)( delayed(_fit_estimator)(clf, X, Y[:, i], sample_weight, ** fit_params_validated) for i, (_, clf) in zip(range(Y.shape[1]), self.estimators)) self.classes_ = [est.classes_ for est in self.estimators_] return self
def _base_est_fit_predict(self, X, y, **fit_params): """Fit the base estimators on CV training folds, and return their out-of-sample predictions on the test folds as features for the meta-estimator. Also return the fit_params for the meta-estimator. """ y = y.squeeze() # Construct CV iterator cv = self._check_cv(y=y) # Extract CV indices since we need them twice, and un-seeded CV # generators with `shuffle=True` split differently each time. train_inds = [] test_inds = [] for train, test in cv.split(X, y): train_inds.append(train) test_inds.append(test) fit_params_ests = self._extract_fit_params(**fit_params) _fit_predict = self._get_fit_predict_function() _jobs = [] # Loop over CV folds to get out-of-sample predictions, which become the # features for the meta-estimator. for train, test in zip(train_inds, test_inds): for name, est in self.estimator_list[:-1]: # adapted from sklearn.model_selection._fit_and_predict # Adjust length of sample weights fit_params_est_adjusted = _check_fit_params( X, fit_params_ests[name], train) # Fit estimator on training set and score out-of-sample _jobs.append( delayed(_fit_predict)(clone(est), X[train], y[train], X[test], **fit_params_est_adjusted)) _out = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, pre_dispatch=self.pre_dispatch)(_jobs) # Extract the results from joblib Xmeta, ymeta = None, None for test in test_inds: ybase = np.empty((y[test].shape[0], 0)) for name, est in self.estimator_list[:-1]: # Build design matrix out of out-of-sample predictions ybase = np.hstack((ybase, _out.pop(0))) # Append the test outputs to what will eventually be the features # for the meta-estimator. if Xmeta is not None: ymeta = np.concatenate((ymeta, y[test])) Xmeta = np.vstack((Xmeta, ybase)) else: Xmeta = ybase ymeta = y[test] return Xmeta, ymeta, fit_params_ests[self.meta_estimator_name]
def test_check_fit_params(indices): X = np.random.randn(4, 2) fit_params = { "list": [1, 2, 3, 4], "array": np.array([1, 2, 3, 4]), "sparse-col": sp.csc_matrix([1, 2, 3, 4]).T, "sparse-row": sp.csc_matrix([1, 2, 3, 4]), "scalar-int": 1, "scalar-str": "xxx", "None": None, } result = _check_fit_params(X, fit_params, indices) indices_ = indices if indices is not None else list(range(X.shape[0])) for key in ["sparse-row", "scalar-int", "scalar-str", "None"]: assert result[key] is fit_params[key] assert result["list"] == _safe_indexing(fit_params["list"], indices_) assert_array_equal(result["array"], _safe_indexing(fit_params["array"], indices_)) assert_allclose_dense_sparse( result["sparse-col"], _safe_indexing(fit_params["sparse-col"], indices_) )
def fit_and_score_te_oracle(estimator, X, y, w, p, t, scorer, train, test, parameters=None, fit_params=None, return_train_score=False, return_parameters=False, return_times=False, return_estimator=False, error_score=np.nan, return_test_score_only=False): """Fit estimator and compute scores for a given dataset split, using oracle knowledge of treatment effects. Based on sklearn.model_selection._validation _fit_and_score, adapted to allow more inputs (treatments and treatment effects) Parameters ---------- estimator : estimator object implementing 'fit' The object to use to fit the data. X : array-like of shape (n_samples, n_features) The features to fit to y : array-like of shape (n_samples,) or (n_samples, ) The outcome variable w: array-like of shape (n_samples,) The treatment indicator p: array-like of shape (n_samples,) The treatment propensity t: array-like of shape (n_samples,) the true treatment effect to evaluate against scorer : A single callable or dict mapping scorer name to the callable If it is a single callable, the return value for ``train_scores`` and ``test_scores`` is a single float. For a dict, it should be one mapping the scorer name to the scorer callable object / function. The callable object / fn should have signature ``scorer(estimator, X, y)``. train : array-like of shape (n_train_samples,) Indices of training samples. test : array-like of shape (n_test_samples,) Indices of test samples. error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit step, which will always raise the error. parameters : dict or None Parameters to be set on the estimator. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. return_train_score : bool, default=False Compute and return score on training set. return_parameters : bool, default=False Return parameters that has been used for the estimator. return_times : bool, default=False Whether to return the fit/score times. return_estimator : bool, default=False Whether to return the fitted estimator. return_test_score_only: bool, default=False Whether to only return a test score Returns ------- train_scores : dict of scorer name -> float Score on training set (for all the scorers), returned only if `return_train_score` is `True`. test_scores : float or dict of scorer name -> float If return_test_score_only and scorer == str, then returns only test score. Otherwise, s on testing set (for all the scorers) n_test_samples : int Number of test samples. fit_time : float Time spent for fitting in seconds. score_time : float Time spent for scoring in seconds. parameters : dict or None The parameters that have been evaluated. estimator : estimator object The fitted estimator """ if not isinstance(estimator, BaseTEModel): raise ValueError("This method works only for BaseTEModel") scorers, _ = _check_multimetric_scoring(estimator, scoring=scorer) # Adjust length of sample weights (if ant) fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) train_scores = {} if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() X_train, y_train, w_train, p_train, t_train = _safe_split_te( X, y, w, p, t, train) X_test, y_test, w_test, p_test, t_test = _safe_split_te( X, y, w, p, t, test) try: estimator.fit(X_train, y_train, w_train, p_train, **fit_params) except Exception as e: if return_test_score_only: if error_score == 'raise': raise else: return np.nan # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exc()), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: fit_time = time.time() - start_time try: test_scores = _score(estimator, X_test, t_test, scorers) except Exception: if return_test_score_only: if error_score == 'raise': raise else: return np.nan score_time = time.time() - start_time - fit_time if return_test_score_only: if type(scorer) == str: return test_scores['score'] else: return test_scores if return_train_score: train_scores = _score(estimator, X_train, t_train, scorers) ret = [train_scores, test_scores] if return_train_score else [test_scores] if return_times: ret.extend([fit_time, score_time]) if return_parameters: ret.append(parameters) if return_estimator: ret.append(estimator) return ret
def fit(self, X, y, groups=None, **fit_params): # sklearn prep cv = check_cv(self.cv, y, classifier=is_classifier(self.estimator)) refit_metric = "score" if callable(self.scoring): scorers = self.scoring elif self.scoring is None or isinstance(self.scoring, str): scorers = check_scoring(self.estimator, self.scoring) else: scorers = _check_multimetric_scoring(self.estimator, self.scoring) # sklearn < 0.24.0 compatibility if isinstance(scorers, tuple): scorers = scorers[0] self._check_refit_for_multimetric(scorers) refit_metric = self.refit X, y, groups = indexable(X, y, groups) fit_params = _check_fit_params(X, fit_params) n_splits = cv.get_n_splits(X, y, groups) base_estimator = clone(self.estimator) rng = check_random_state(self.random_state) np.random.set_state(rng.get_state(legacy=True)) np_random_seed = rng.get_state(legacy=True)[1][0] n_jobs, actual_iterations = self._calculate_n_jobs_and_actual_iters() # default port is 9090, we must have one, this is how BOHB workers communicate (even locally) run_id = f"HpBandSterSearchCV_{time.time()}" _nameserver = hpns.NameServer(run_id=run_id, host=self.nameserver_host, port=self.nameserver_port) gc.collect() if self.verbose > 1: _logger.setLevel(logging.DEBUG) elif self.verbose > 0: _logger.setLevel(logging.INFO) else: _logger.setLevel(logging.ERROR) if "logger" in self.bohb_kwargs: self.bohb_kwargs.pop("logger") with NameServerContext(_nameserver): workers = [] # each worker is a separate thread for i in range(n_jobs): # SklearnWorker clones the estimator w = SklearnWorker( min_budget=self.min_budget, max_budget=self.max_budget, base_estimator=self.estimator, X=X, y=y, cv=cv, cv_n_splits=n_splits, groups=groups, scoring=scorers, metric=refit_metric, fit_params=fit_params, nameserver=self.nameserver_host, nameserver_port=self.nameserver_port, run_id=run_id, id=i, return_train_score=self.return_train_score, error_score=self.error_score, resource_name=self.resource_name, resource_type=self.resource_type, random_state=rng, logger=_logger, ) w.run(background=True) workers.append(w) converted_min_budget = float(workers[0].min_budget) converted_max_budget = float(workers[0].max_budget) self.resource_name_ = workers[0].resource_name if (self.resource_name_ in self.param_distributions.get_hyperparameter_names()): _logger.warning( f"Found hyperparameter with name '{self.resource_name_}', same as resource_name_. Removing it from ConfigurationSpace." ) param_distributions = CS.ConfigurationSpace( name=self.param_distributions.name, meta=self.param_distributions.meta, ) param_distributions.add_hyperparameters([ x for x in self.param_distributions.get_hyperparameters() if x.name != self.resource_name_ ]) else: param_distributions = deepcopy(self.param_distributions) param_distributions.seed = np_random_seed # sleep for a moment to make sure all workers are initialized sleep(0.2) # BOHB by default if isinstance(self.optimizer, str): optimizer = self._optimizer_dict[self.optimizer.lower()]( configspace=param_distributions, run_id=run_id, min_budget=converted_min_budget, max_budget=converted_max_budget, logger=_logger, **self.bohb_kwargs, ) else: optimizer = self.optimizer( configspace=param_distributions, run_id=run_id, min_budget=converted_min_budget, max_budget=converted_max_budget, logger=_logger, **self.bohb_kwargs, ) with OptimizerContext( optimizer, n_iterations=actual_iterations, ) as res: self._res = res id2config = self._res.get_id2config_mapping() incumbent = self._res.get_incumbent_id() runs_all = self._res.get_all_runs() self.best_params_ = id2config[incumbent]["config"] resource_type = workers[0].resource_type self.n_resources_ = [resource_type(x) for x in optimizer.budgets] self.min_resources_ = self.n_resources_[0] self.max_resources_ = self.n_resources_[-1] results, new_refit_metric = self._runs_to_results( runs_all, id2config, scorers, n_splits, self.n_resources_) if new_refit_metric is not None: refit_metric = new_refit_metric iter_counter = sorted(Counter(results["iter"]).items()) self.n_candidates_ = [x[1] for x in iter_counter] self.n_remaining_candidates_ = iter_counter[-1][1] self.n_iterations_ = iter_counter[-1][0] + 1 # For multi-metric evaluation, store the best_index_, best_params_ and # best_score_ iff refit is one of the scorer names # In single metric evaluation, refit_metric is "score" if self.refit or not self.multimetric_: # If callable, refit is expected to return the index of the best # parameter set. if callable(self.refit): self.best_index_ = self.refit(results) if not isinstance(self.best_index_, numbers.Integral): raise TypeError("best_index_ returned is not an integer") if self.best_index_ < 0 or self.best_index_ >= len( results["params"]): raise IndexError("best_index_ index out of range") else: self.best_index_ = results["rank_test_%s" % refit_metric].argmin() self.best_score_ = results["mean_test_%s" % refit_metric][self.best_index_] self.best_params_ = results["params"][self.best_index_] _logger.info( f"\nBest {refit_metric}: {self._res.get_runs_by_id(incumbent)[-1].info['test_score_mean']}" ) _logger.info(f"Best found configuration: {self.best_params_}") _logger.info( f"A total of {len(id2config.keys())} unique configurations where sampled." ) _logger.info(f"A total of {len(runs_all)} runs where executed.") _logger.info( f"Total budget of resource '{self.resource_name_}' corresponds to {sum([r.budget for r in runs_all]) / converted_max_budget} full function evaluations." ) gc.collect() if self.refit: # we clone again after setting params in case some # of the params are estimators as well. refit_params = self.best_params_.copy() if self.resource_name_ != "n_samples": refit_params[self.resource_name_] = self.max_resources_ self.best_estimator_ = clone( clone(base_estimator).set_params(**refit_params)) refit_start_time = time.time() if y is not None: self.best_estimator_.fit(X, y, **fit_params) else: self.best_estimator_.fit(X, **fit_params) refit_end_time = time.time() self.refit_time_ = refit_end_time - refit_start_time # Store the only scorer not as a dict for single metric evaluation self.scorer_ = scorers self.cv_results_ = results self.n_splits_ = n_splits return self
def _fit_and_score_open_set(estimator, X_train, y_train, X_test, y_test, scorer, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, return_estimator=False, error_score=np.nan): """ Adapts the method sklearn.model_selection._validation._fit_and_score to the open-set setting, where some labels correspond to unknown camera models. Fit estimator and compute scores for a given dataset split. Parameters ---------- estimator : estimator object implementing 'fit' The object to use to fit the data. X_train : array-like of shape (n_samples, n_features) The data to fit. y_train : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The target variable to try to predict in the case of supervised learning. X_test : array-like of shape (n_samples, n_features) The test data y_test : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The test labels scorer : A single callable or dict mapping scorer name to the callable If it is a single callable, the return value for ``train_scores`` and ``test_scores`` is a single float. For a dict, it should be one mapping the scorer name to the scorer callable object / function. The callable object / fn should have signature ``scorer(estimator, X, y)``. verbose : int The verbosity level. error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit step, which will always raise the error. parameters : dict or None Parameters to be set on the estimator. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. return_train_score : bool, default=False Compute and return score on training set. return_parameters : bool, default=False Return parameters that has been used for the estimator. return_n_test_samples : bool, default=False Whether to return the ``n_test_samples`` return_times : bool, default=False Whether to return the fit/score times. return_estimator : bool, default=False Whether to return the fitted estimator. Returns ------- train_scores : dict of scorer name -> float Score on training set (for all the scorers), returned only if `return_train_score` is `True`. test_scores : dict of scorer name -> float Score on testing set (for all the scorers). n_test_samples : int Number of test samples. fit_time : float Time spent for fitting in seconds. score_time : float Time spent for scoring in seconds. parameters : dict or None The parameters that have been evaluated. estimator : estimator object The fitted estimator """ if verbose > 1: if parameters is None: msg = '' else: msg = '%s' % (', '.join('%s=%s' % (k, v) for k, v in parameters.items())) print("[CV] %s %s" % (msg, (64 - len(msg)) * '.')) # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X_train, fit_params) train_scores = {} if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() try: if y_train is None: estimator.fit(X_train, **fit_params) else: estimator.fit(X_train, y_train, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exc()), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: fit_time = time.time() - start_time test_scores = _score(estimator, X_test, y_test, scorer) score_time = time.time() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer) if verbose > 2: if isinstance(test_scores, dict): for scorer_name in sorted(test_scores): msg += ", %s=" % scorer_name if return_train_score: msg += "(train=%.3f," % train_scores[scorer_name] msg += " test=%.3f)" % test_scores[scorer_name] else: msg += "%.3f" % test_scores[scorer_name] else: msg += ", score=" msg += ("%.3f" % test_scores if not return_train_score else "(train=%.3f, test=%.3f)" % (train_scores, test_scores)) if verbose > 1: total_time = score_time + fit_time print(_message_with_time('CV', msg, total_time)) ret = [train_scores, test_scores] if return_train_score else [test_scores] if return_n_test_samples: ret.append(_num_samples(X_test)) if return_times: ret.extend([fit_time, score_time]) if return_parameters: ret.append(parameters) if return_estimator: ret.append(estimator) return ret
def fp_fit_and_score(estimator, X_original, y_original, X_fingerprinted, y_fingerprinted, scorer, train_original, test_original, train_fingerprinted, test_fingerprinted, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, return_estimator=False, split_progress=None, candidate_progress=None, error_score=np.nan): if not isinstance(error_score, numbers.Number) and error_score != 'raise': raise ValueError( "error_score must be the string 'raise' or a numeric value. " "(Hint: if using 'raise', please make sure that it has been " "spelled correctly.)") progress_msg = "" if verbose > 2: if split_progress is not None: progress_msg = f" {split_progress[0]+1}/{split_progress[1]}" if candidate_progress and verbose > 9: progress_msg += (f"; {candidate_progress[0]+1}/" f"{candidate_progress[1]}") if verbose > 1: if parameters is None: params_msg = '' else: sorted_keys = sorted(parameters) # Ensure deterministic o/p params_msg = (', '.join(f'{k}={parameters[k]}' for k in sorted_keys)) if verbose > 9: start_msg = f"[CV{progress_msg}] START {params_msg}" print(f"{start_msg}{(80 - len(start_msg)) * '.'}") # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X_fingerprinted, fit_params, train_fingerprinted) if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() # here I need to make sure to split the fingerprinted data IN THE SAME WAY # original train data should be unused # fingerprinted test data should be unused X_train_original, y_train_original = _safe_split(estimator, X_original, y_original, train_original) X_test_original, y_test_original = _safe_split(estimator, X_original, y_original, test_original, train_original) X_train_fingerprinted, y_train_fingerprinted = _safe_split( estimator, X_fingerprinted, y_fingerprinted, train_fingerprinted) X_test_fingerprinted, y_test_fingerprinted = _safe_split( estimator, X_fingerprinted, y_fingerprinted, test_fingerprinted, train_fingerprinted) result = {} # fit the model on FINGERPRINTED data try: if y_train_fingerprinted is None: estimator.fit(X_train_fingerprinted, **fit_params) else: estimator.fit(X_train_fingerprinted, y_train_fingerprinted, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exc()), FitFailedWarning) result["fit_failed"] = True else: result["fit_failed"] = False fit_time = time.time() - start_time # obtain test scores from testing ORIGINAL test data against ORIGINAL target test_scores = _score(estimator, X_test_original, y_test_original, scorer, error_score) # VERIFICATION PRINTOUTS # print(len(X_train_original.index)) # print(len(X_train_fingerprinted.index)) # print(type(X_test_original.index)) # print(X_train_original.index) # print(X_train_fingerprinted.index) # print(X_train_original.index.equals(X_train_fingerprinted.index)) # print(X_train_original.columns[1]) # print(X_train_original[X_train_original.columns[1]].compare # (X_train_fingerprinted[X_train_fingerprinted.columns[1]])) # print('----------------') # print(X_test_original[X_test_original.columns[1]].compare(X_test_fingerprinted[X_test_fingerprinted.columns[1]])) # print('________________') # print('Target should look the same') # print(y_train_fingerprinted.compare(y_train_original)) # print('________________') score_time = time.time() - start_time - fit_time if return_train_score: # train scores are based on FINGERPRINTED data train_scores = _score(estimator, X_train_fingerprinted, y_train_fingerprinted, scorer, error_score) if verbose > 1: total_time = score_time + fit_time end_msg = f"[CV{progress_msg}] END " result_msg = params_msg + (";" if params_msg else "") if verbose > 2 and isinstance(test_scores, dict): for scorer_name in sorted(test_scores): result_msg += f" {scorer_name}: (" if return_train_score: scorer_scores = train_scores[scorer_name] result_msg += f"train={scorer_scores:.3f}, " result_msg += f"test={test_scores[scorer_name]:.3f})" result_msg += f" total time={logger.short_format_time(total_time)}" # Right align the result_msg end_msg += "." * (80 - len(end_msg) - len(result_msg)) end_msg += result_msg print(end_msg) result["test_scores"] = test_scores if return_train_score: result["train_scores"] = train_scores if return_n_test_samples: result["n_test_samples"] = _num_samples(X_test_original) if return_times: result["fit_time"] = fit_time result["score_time"] = score_time if return_parameters: result["parameters"] = parameters if return_estimator: result["estimator"] = estimator return result
def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, return_estimator=False, split_progress=None, candidate_progress=None, error_score=np.nan, online_train_val_split=False): """Fit estimator and compute scores for a given dataset split. Parameters ---------- estimator : estimator object implementing 'fit' The object to use to fit the data. X : array-like of shape (n_samples, n_features) The data to fit. y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The target variable to try to predict in the case of supervised learning. scorer : A single callable or dict mapping scorer name to the callable If it is a single callable, the return value for ``train_scores`` and ``test_scores`` is a single float. For a dict, it should be one mapping the scorer name to the scorer callable object / function. The callable object / fn should have signature ``scorer(estimator, X, y)``. train : array-like of shape (n_train_samples,) Indices of training samples. test : array-like of shape (n_test_samples,) Indices of test samples. verbose : int The verbosity level. error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. parameters : dict or None Parameters to be set on the estimator. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. return_train_score : bool, default=False Compute and return score on training set. return_parameters : bool, default=False Return parameters that has been used for the estimator. split_progress : {list, tuple} of int, default=None A list or tuple of format (<current_split_id>, <total_num_of_splits>). candidate_progress : {list, tuple} of int, default=None A list or tuple of format (<current_candidate_id>, <total_number_of_candidates>). return_n_test_samples : bool, default=False Whether to return the ``n_test_samples``. return_times : bool, default=False Whether to return the fit/score times. return_estimator : bool, default=False Whether to return the fitted estimator. Returns ------- result : dict with the following attributes train_scores : dict of scorer name -> float Score on training set (for all the scorers), returned only if `return_train_score` is `True`. test_scores : dict of scorer name -> float Score on testing set (for all the scorers). n_test_samples : int Number of test samples. fit_time : float Time spent for fitting in seconds. score_time : float Time spent for scoring in seconds. parameters : dict or None The parameters that have been evaluated. estimator : estimator object The fitted estimator. fit_failed : bool The estimator failed to fit. """ if not isinstance(error_score, numbers.Number) and error_score != 'raise': raise ValueError( "error_score must be the string 'raise' or a numeric value. " "(Hint: if using 'raise', please make sure that it has been " "spelled correctly.)") progress_msg = "" if verbose > 2: if split_progress is not None: progress_msg = f" {split_progress[0]+1}/{split_progress[1]}" if candidate_progress and verbose > 9: progress_msg += (f"; {candidate_progress[0]+1}/" f"{candidate_progress[1]}") if verbose > 1: if parameters is None: params_msg = '' else: sorted_keys = sorted(parameters) # Ensure deterministic o/p params_msg = (', '.join(f'{k}={parameters[k]}' for k in sorted_keys)) if verbose > 9: start_msg = f"[CV{progress_msg}] START {params_msg}" print(f"{start_msg}{(80 - len(start_msg)) * '.'}") # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() if online_train_val_split: # inject the train and test data into the corresponding Subset selectors set_train_estim = False set_test_estim = False for estim in estimator: if set_train_estim and set_test_estim: break if isinstance(estim, TrainSubset): estim.date_range = [train] set_train_estim = True if isinstance(estim, TestSubset): estim.date_range = [test] set_test_estim = True if isinstance(estim, CVSubset) and isinstance( estim, EvalOnlyWrapper): estim.set_range(test) set_test_estim = True if isinstance(estim, CVSubset) and isinstance( estim, TrainOnlyWrapper): estim.set_range(train) set_train_estim = True if not set_train_estim or not set_test_estim: raise ValueError( "when specifying online learning a KeepTrain and KeepTest have to be in the pipeline" ) else: X_train, y_train = _safe_split(estimator, X, y, train) X_test, y_test = _safe_split(estimator, X, y, test, train) result = {} try: if online_train_val_split: estimator = estimator.train() estimator.fit(X, y, **fit_params) else: if y_train is None: estimator.fit(X_train, **fit_params) else: estimator.fit(X_train, y_train, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exc()), FitFailedWarning) result["fit_failed"] = True y_sample_len = len(test) else: result["fit_failed"] = False fit_time = time.time() - start_time estimator.eval() if online_train_val_split: # select estimator without the classifier and transform x and y # to retrieve y_test _, y_prime = estimator[:-1].transform(X, y) if isinstance(y_prime, pd.DataFrame) and len(y_prime.columns) == 1: y_prime = y_prime.T.values.squeeze() y_sample_len = len(y_prime) test_scores = _score(estimator, X, y_prime, scorer) else: test_scores = _score(estimator, X_test, y_test, scorer, error_score) score_time = time.time() - start_time - fit_time if return_train_score: if online_train_val_split: estimator.train() _, y_prime = estimator[:-1].transform(X, y) if isinstance(y_prime, pd.DataFrame) and len( y_prime.columns) == 1: y_prime = y_prime.T.values.squeeze() train_scores = _score(estimator, X, y_prime, scorer) estimator.eval() else: train_scores = _score(estimator, X_train, y_train, scorer, error_score) if verbose > 1: total_time = score_time + fit_time end_msg = f"[CV{progress_msg}] END " result_msg = params_msg + (";" if params_msg else "") if verbose > 2 and isinstance(test_scores, dict): for scorer_name in sorted(test_scores): result_msg += f" {scorer_name}: (" if return_train_score: scorer_scores = train_scores[scorer_name] result_msg += f"train={scorer_scores:.3f}, " result_msg += f"test={test_scores[scorer_name]:.3f})" result_msg += f" total time={logger.short_format_time(total_time)}" # Right align the result_msg end_msg += "." * (80 - len(end_msg) - len(result_msg)) end_msg += result_msg print(end_msg) result["test_scores"] = test_scores if return_train_score: result["train_scores"] = train_scores if return_n_test_samples: if online_train_val_split: result["n_test_samples"] = y_sample_len else: result["n_test_samples"] = _num_samples(X_test) if return_times: result["fit_time"] = fit_time result["score_time"] = score_time if return_parameters: result["parameters"] = parameters if return_estimator: result["estimator"] = estimator return result
def _fit_and_predict(estimator, X, y, sample_weight, sample_weight_steps, train, test, verbose, fit_params, method): """Fit estimator and predict values for a given dataset split. Read more in the :ref:`User Guide <cross_validation>`. Parameters ---------- estimator : estimator object implementing 'fit' and 'predict' The object to use to fit the data. X : array-like of shape at least 2D The data to fit. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. sample_weight: array-like, optional sample_weight_steps: array-like, optional train : array-like, shape (n_train_samples,) Indices of training samples. test : array-like, shape (n_test_samples,) Indices of test samples. verbose : integer The verbosity level. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. method : string Invokes the passed method name of the passed estimator. Returns ------- predictions : sequence Result of calling 'estimator.method' test : array-like This is the value of the test parameter """ # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) X_train, y_train, sample_weight_train = weighted_safe_split(estimator, X, y, sample_weight, train) X_test, _, _ = weighted_safe_split(estimator, X, y, sample_weight, test, train) if y_train is None and sample_weight_train is None: estimator.fit(X_train, **fit_params) elif sample_weight_train is None: estimator.fit(X_train, y_train, **fit_params) else: if sample_weight_steps is not None: for step in sample_weight_steps: fit_params[step + '__sample_weight'] = sample_weight_train else: fit_params['sample_weight'] = sample_weight_train estimator.fit(X_train, y_train, **fit_params) func = getattr(estimator, method) predictions = func(X_test) if method in ['decision_function', 'predict_proba', 'predict_log_proba']: n_classes = len(set(y)) if n_classes != len(estimator.classes_): recommendation = ( 'To fix this, use a cross-validation ' 'technique resulting in properly ' 'stratified folds') warnings.warn('Number of classes in training fold ({}) does ' 'not match total number of classes ({}). ' 'Results may not be appropriate for your use case. ' '{}'.format(len(estimator.classes_), n_classes, recommendation), RuntimeWarning) if method == 'decision_function': if (predictions.ndim == 2 and predictions.shape[1] != len(estimator.classes_)): # This handles the case when the shape of predictions # does not match the number of classes used to train # it with. This case is found when sklearn.svm.SVC is # set to `decision_function_shape='ovo'`. raise ValueError('Output shape {} of {} does not match ' 'number of classes ({}) in fold. ' 'Irregular decision_function outputs ' 'are not currently supported by ' 'cross_val_predict'.format( predictions.shape, method, len(estimator.classes_), recommendation)) if len(estimator.classes_) <= 2: # In this special case, `predictions` contains a 1D array. raise ValueError('Only {} class/es in training fold, this ' 'is not supported for decision_function ' 'with imbalanced folds. {}'.format( len(estimator.classes_), recommendation)) float_min = np.finfo(predictions.dtype).min default_values = {'decision_function': float_min, 'predict_log_proba': float_min, 'predict_proba': 0} predictions_for_all_classes = np.full((_num_samples(predictions), n_classes), default_values[method]) predictions_for_all_classes[:, estimator.classes_] = predictions predictions = predictions_for_all_classes return predictions, test
def _fitnscore_cust(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, return_n_iter=False, return_cm=False, return_roc=False, return_prc=False, return_threshc=False, return_estimator=False, error_score=np.nan): # estimator, scorer, return_train_score, return_parameters, return_n_test_samples, return_times, return_n_iter, return_cm, return_roc, return_prc, return_threshc, return_estimator, error_score, verbose = clone(base_estimator), scorers, True, True, True, True, True, True, True, True, True, True, np.nan, 5 # parameters, (train, test) = next(product(candidate_params, cv.split(X, y, groups))) import numbers from traceback import format_exc from sklearn.utils.validation import _check_fit_params, _num_samples from sklearn.utils.metaestimators import _safe_split from sklearn.exceptions import FitFailedWarning from sklearn.model_selection._validation import _score from sklearn.utils import _message_with_time if verbose > 1: if parameters is None: msg = '' else: msg = '%s' % (', '.join('%s=%s' % (k, v) for k, v in parameters.items())) print("[CV] %s %s" % (msg, (64 - len(msg)) * '.')) # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) train_scores = {} if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() X_train, y_train = _safe_split(estimator, X, y, train) X_test, y_test = _safe_split(estimator, X, y, test, train) try: with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=ConvergenceWarning) if y_train is None: estimator.fit(X_train, **fit_params) else: estimator.fit(X_train, y_train, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exc()), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: fit_time = time.time() - start_time test_scores = _score(estimator, X_test, y_test, scorer) score_time = time.time() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer) if isinstance(estimator, pipeline.Pipeline): n_iter = estimator.steps[-1][1].n_iter_ else: n_iter = estimator.n_iter_ if return_cm: cm = dict( zip([ 'true_negatives', 'false_positives', 'false_negatives', 'true_positives' ], [0] * 4)) for true, pred in zip(y_test, estimator.predict(X_test)): # true, pred = next(zip(y_test, estimator.predict(X_test))) s = sum([true, pred]) if s == 2: cm['true_positives'] += 1 elif s == 0: cm['true_negatives'] += 1 elif true == 1: cm['false_negatives'] += 1 elif pred == 1: cm['false_positives'] += 1 else: raise Exception(f"Values {true} and {pred} not valid for" " computing confusion matrix") if return_roc or return_prc or return_threshc: df = estimator.decision_function(X_test) if return_roc: roc = metrics.roc_curve(y_test, df) auc = metrics.auc(roc[1], roc[2]) if return_threshc or return_prc: prc = metrics.precision_recall_curve(y_test, df) ap = metrics.average_precision_score(y_test, df) if return_roc: roc_out = {'fpr': roc[0], 'tpr': roc[1], 'auc': auc} if return_prc: prc_out = {'rcl': prc[1], 'pre': prc[0], 'ap': ap} if return_threshc: threshc_out = { 'thr': prc[2], 'rcl': prc[1][:-1], 'pre': prc[0][:-1] } if verbose > 2: if isinstance(test_scores, dict): for scorer_name in sorted(test_scores): msg += ", %s=" % scorer_name if return_train_score: msg += "(train=%.3f," % train_scores[scorer_name] msg += " test=%.3f)" % test_scores[scorer_name] else: msg += "%.3f" % test_scores[scorer_name] else: msg += ", score=" msg += ("%.3f" % test_scores if not return_train_score else "(train=%.3f, test=%.3f)" % (train_scores, test_scores)) if verbose > 1: total_time = score_time + fit_time print(_message_with_time('CV', msg, total_time)) ret = {'test_scores': test_scores} if return_train_score: ret['train_scores'] = train_scores if return_n_test_samples: ret['n_test_samples'] = _num_samples(X_test) if return_times: ret['fit_time'] = fit_time ret['score_time'] = score_time if return_parameters: ret['parameters'] = parameters if return_n_iter: ret['n_iter'] = n_iter if return_estimator: ret['estimator'] = estimator if return_cm: ret['confusion_matrix'] = cm if return_roc: ret['roc_values'] = roc_out if return_prc: ret['prc_values'] = prc_out if return_threshc: ret['threshc_values'] = threshc_out return ret
def fit(self, X, y=None, *, groups=None, **fit_params): """Run fit with all sets of parameters. Parameters ---------- X : array-like of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. y : array-like of shape (n_samples, n_output) \ or (n_samples,), default=None Target relative to X for classification or regression; None for unsupervised learning. groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`~sklearn.model_selection.GroupKFold`). **fit_params : dict of str -> object Parameters passed to the ``fit`` method of the estimator """ # self, X, y, groups, fit_params = gs, train, cls, None, {} estimator = self.estimator cv = check_cv(self.cv, y, classifier=is_classifier(estimator)) scorers, self.multimetric_ = _check_multimetric_scoring( self.estimator, scoring=self.scoring) if self.multimetric_: if self.refit is not False and ( not isinstance(self.refit, str) or # This will work for both dict / list (tuple) self.refit not in scorers) and not callable(self.refit): raise ValueError("For multi-metric scoring, the parameter " "refit must be set to a scorer key or a " "callable to refit an estimator with the " "best parameter setting on the whole " "data and make the best_* attributes " "available for that metric. If this is " "not needed, refit should be set to " "False explicitly. %r was passed." % self.refit) else: refit_metric = self.refit else: refit_metric = 'score' X, y, groups = indexable(X, y, groups) fit_params = _check_fit_params(X, fit_params) n_splits = cv.get_n_splits(X, y, groups) base_estimator = clone(self.estimator) parallel = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, pre_dispatch=self.pre_dispatch) fit_and_score_kwargs = dict(scorer=scorers, fit_params=fit_params, return_train_score=self.return_train_score, return_n_test_samples=True, return_times=True, return_parameters=False, return_n_iter=True, return_cm=True, return_roc=True, return_prc=True, return_threshc=True, return_estimator=False, error_score=self.error_score, verbose=self.verbose) results = {} with parallel: all_candidate_params = [] all_out = [] def evaluate_candidates(candidate_params): # candidate_params = model_selection.ParameterGrid(self.param_grid) candidate_params = list(candidate_params) n_candidates = len(candidate_params) if self.verbose > 0: print("Fitting {0} folds for each of {1} candidates," " totalling {2} fits".format( n_splits, n_candidates, n_candidates * n_splits)) out = parallel( delayed(_fitnscore_cust)(clone(base_estimator), X, y, train=train, test=test, parameters=parameters, **fit_and_score_kwargs) for parameters, (train, test) in product( candidate_params, cv.split(X, y, groups))) if len(out) < 1: raise ValueError('No fits were performed. ' 'Was the CV iterator empty? ' 'Were there no candidates?') elif len(out) != n_candidates * n_splits: raise ValueError('cv.split and cv.get_n_splits returned ' 'inconsistent results. Expected {} ' 'splits, got {}'.format( n_splits, len(out) // n_candidates)) all_candidate_params.extend(candidate_params) all_out.extend(out) nonlocal results results = self._format_results(all_candidate_params, scorers, n_splits, all_out) return results self._run_search(evaluate_candidates) # For multi-metric evaluation, store the best_index_, best_params_ and # best_score_ iff refit is one of the scorer names # In single metric evaluation, refit_metric is "score" if self.refit or not self.multimetric_: # If callable, refit is expected to return the index of the best # parameter set. if callable(self.refit): self.best_index_ = self.refit(results) if not isinstance(self.best_index_, numbers.Integral): raise TypeError('best_index_ returned is not an integer') if (self.best_index_ < 0 or self.best_index_ >= len(results["params"])): raise IndexError('best_index_ index out of range') else: self.best_index_ = results["rank_test_%s" % refit_metric].argmin() self.best_score_ = results["mean_test_%s" % refit_metric][self.best_index_] self.best_params_ = results["params"][self.best_index_] if self.refit: # we clone again after setting params in case some # of the params are estimators as well. self.best_estimator_ = clone( clone(base_estimator).set_params(**self.best_params_)) refit_start_time = time.time() if y is not None: self.best_estimator_.fit(X, y, **fit_params) else: self.best_estimator_.fit(X, **fit_params) refit_end_time = time.time() self.refit_time_ = refit_end_time - refit_start_time # Store the only scorer not as a dict for single metric evaluation self.scorer_ = scorers if self.multimetric_ else scorers['score'] self.cv_results_ = results self.n_splits_ = n_splits return self
def fit(self, X, y=None, groups=None, **fit_params): """Run fit with all sets of parameters. Parameters ---------- X : array-like of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. y : array-like of shape (n_samples, n_output) or (n_samples,), optional Target relative to X for classification or regression; None for unsupervised learning. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`~sklearn.model_selection.GroupKFold`). **fit_params : dict of string -> object Parameters passed to the ``fit`` method of the estimator """ estimator = self.estimator cv = check_cv(self.cv, y, classifier=is_classifier(estimator)) scorers, self.multimetric_ = _check_multimetric_scoring( self.estimator, scoring=self.scoring) if self.multimetric_: if self.refit is not False and ( not isinstance(self.refit, str) or # This will work for both dict / list (tuple) self.refit not in scorers) and not callable(self.refit): raise ValueError("For multi-metric scoring, the parameter " "refit must be set to a scorer key or a " "callable to refit an estimator with the " "best parameter setting on the whole " "data and make the best_* attributes " "available for that metric. If this is " "not needed, refit should be set to " "False explicitly. %r was passed." % self.refit) else: refit_metric = self.refit else: refit_metric = 'score' X, y, groups = indexable(X, y, groups) fit_params = _check_fit_params(X, fit_params) n_splits = cv.get_n_splits(X, y, groups) base_estimator = clone(self.estimator) # handled in fit functions below # parallel = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, # pre_dispatch=self.pre_dispatch) fit_and_score_kwargs = dict(scorer=scorers, fit_params=fit_params, return_train_score=self.return_train_score, return_n_test_samples=True, return_times=True, return_parameters=False, error_score=self.error_score, verbose=self.verbose) # sklearn code above if self.spark is None: fitting_function = self._run_sklearn_fit else: fitting_function = self._run_skspark_fit results = fitting_function(base_estimator=base_estimator, X=X, y=y, scorers=scorers, cv=cv, groups=groups, n_splits=n_splits, fit_and_score_kwargs=fit_and_score_kwargs) # sklearn code below # For multi-metric evaluation, store the best_index_, best_params_ and # best_score_ iff refit is one of the scorer names # In single metric evaluation, refit_metric is "score" if self.refit or not self.multimetric_: # If callable, refit is expected to return the index of the best # parameter set. if callable(self.refit): self.best_index_ = self.refit(results) if not isinstance(self.best_index_, numbers.Integral): raise TypeError('best_index_ returned is not an integer') if (self.best_index_ < 0 or self.best_index_ >= len(results["params"])): raise IndexError('best_index_ index out of range') else: self.best_index_ = results["rank_test_%s" % refit_metric].argmin() self.best_score_ = results["mean_test_%s" % refit_metric][self.best_index_] self.best_params_ = results["params"][self.best_index_] if self.refit: # we clone again after setting params in case some # of the params are estimators as well. self.best_estimator_ = clone( clone(base_estimator).set_params(**self.best_params_)) refit_start_time = time.time() if y is not None: self.best_estimator_.fit(X, y, **fit_params) else: self.best_estimator_.fit(X, **fit_params) refit_end_time = time.time() self.refit_time_ = refit_end_time - refit_start_time # Store the only scorer not as a dict for single metric evaluation self.scorer_ = scorers if self.multimetric_ else scorers['score'] self.cv_results_ = results self.n_splits_ = n_splits return self
def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, *, groups: Optional[np.ndarray] = None, **fit_params: dict) -> SHGOSearchCV: """ Run the optimization based on the parameters defined before. Parameters ---------- X : np.ndarray of shape (n_samples, n_features) Training vector, where `n_samples` is the number of samples and `n_features` is the number of features. y : np.ndarray of shape(n_samples, n_output) or (n_samples, ), default = None Target relative to X for classification or regression; None for unsupervised learning. groups : np.ndarray of shape(n_samples, ), default = None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`~sklearn.model_selection.GroupKFold`). **fit_params : dict of str -> object Parameters passed to the ``fit`` method of the estimator. Returns ------- self : object Instance of fitted estimator. """ estimator = self.estimator func = self.func X, y, groups = indexable(X, y, groups) fit_params = _check_fit_params(X, fit_params) cv_orig = check_cv(self.cv, y, classifier=is_classifier(estimator)) n_splits = cv_orig.get_n_splits(X, y, groups) base_estimator = clone(self.estimator) param_names = sorted(self.params) bounds = [self.params[name] for name in param_names] constraints = self.constraints train = [None] * n_splits test = [None] * n_splits for k, (tr, te) in enumerate(self.cv.split(X, y, groups)): train[k] = tr test[k] = te res = optimize.shgo(func=func, bounds=bounds, constraints=constraints, args=(param_names, clone(base_estimator), X, y, train, test)) result = {} for param_name, param_value in zip(param_names, res.x): result[param_name] = param_value self.best_params_ = result if self.refit: # we clone again after setting params in case some # of the params are estimators as well. self.best_estimator_ = clone( clone(base_estimator).set_params(**self.best_params_)) refit_start_time = time.time() if y is not None: self.best_estimator_.fit(X, y, **fit_params) else: self.best_estimator_.fit(X, **fit_params) refit_end_time = time.time() self.refit_time_ = refit_end_time - refit_start_time if hasattr(self.best_estimator_, "feature_names_in_"): self.feature_names_in_ = self.best_estimator_.feature_names_in_ self.n_splits_ = n_splits return self
def fit(self, X, y=None, **fit_params): cv = check_cv(self.cv, y, classifier=is_classifier(self)) fit_params = _check_fit_params(X, fit_params) scorers, self.multimetric_ = _check_multimetric_scoring( self, scoring=self.scoring) best_scorer = list(scorers.keys())[0] self.results_ = {} self.summary = pd.DataFrame(columns=['model', 'params'] + [ f'{prefix} {scoring}' for scoring in scorers.keys() for prefix in ('mean', 'std') ]) self.estimators_ = [] estimators, params = self._validate_estimators() for i in range(len(estimators)): prefix = '' _params = params[i].copy() _name = estimators[i].__class__.__name__ if self.verbose: print( _name, ''.join(f'\n {name}: {str(v)}' for name, v in _params.items())) search = _params.pop('search', 'grid') n_iter = _params.pop('n_iter', 5) if is_classifier(self): prefix = 'base_estimator__' estimators[i] = CalibratedClassifierCV( base_estimator=estimators[i]) _prefixed_params = { prefix + name: v for name, v in _params.items() } if search == 'grid': estimator_ = GridSearchCV(estimators[i], _prefixed_params, scoring=self.scoring, refit=best_scorer, cv=self.cv, n_jobs=self.n_jobs).fit(X, y) else: estimator_ = RandomizedSearchCV( estimators[i], _prefixed_params, scoring=self.scoring, refit=best_scorer, cv=self.cv, n_jobs=self.n_jobs, n_iter=n_iter, random_state=self.random_state).fit(X, y) self.estimators_.append(estimator_.best_estimator_) for name, v in estimator_.cv_results_.items(): v = v[estimator_.best_index_] v = {key[len(prefix):]: value for key, value in v.items()} if name == 'params' else v if name in self.results_: self.results_[name] = np.append(self.results_[name], v) if name[:4] == 'rank': score_rank = self.results_['mean' + name[4:]] self.results_[name] = len(score_rank) - np.argsort( score_rank) else: self.results_[name] = np.array([v]) summary = { 'model': _name, 'params': '; '.join(f'{name}: {v}' for name, v in self.results_['params'][-1].items()) } for scorer in scorers.keys(): summary['mean ' + scorer] = self.results_['mean_test_' + scorer][-1] summary['std ' + scorer] = self.results_['std_test_' + scorer][-1] self.summary = self.summary.append(pd.Series(summary), ignore_index=True) self.best_index_ = np.argmin(self.results_['rank_test_' + best_scorer]) self.best_score_ = self.results_['rank_test_' + best_scorer][self.best_index_] self.best_estimator_ = self.estimators_[self.best_index_] if self.verbose: print('\nBest score:') for name, v in self.results_.items(): if name[:9] == 'mean_test': print(' {}: {:.4f}'.format(name[10:], v[-1]), end=' ') elif name[:8] == 'std_test': print('+/- {:.4f}'.format(v[-1])) print('=' * 50, end='\n\n') self.scorer_ = scorers if self.multimetric_ else scorers['score'] if self.verbose: print('Best model:') print(f" {self.summary['model'].iloc[self.best_index_]}") print('\n'.join(f' {key}: {str(value)}' for key, value in self.results_['params'][self.best_index_].items())) for key, value in self.results_.items(): if key[:9] == 'mean_test': print(' {}: {:.4f}'.format(key[10:], value[self.best_index_]), end=' ') elif key[:8] == 'std_test': print('+/- {:.4f}'.format(value[self.best_index_])) return self
def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, return_estimator=False, split_progress=None, candidate_progress=None, error_score=np.nan): """Fit estimator and compute scores for a given dataset split. Parameters ---------- estimator : estimator object implementing 'fit' The object to use to fit the data. X : array-like of shape (n_samples, n_features) The data to fit. y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The target variable to try to predict in the case of supervised learning. scorer : A single callable or dict mapping scorer name to the callable If it is a single callable, the return value for ``train_scores`` and ``test_scores`` is a single float. For a dict, it should be one mapping the scorer name to the scorer callable object / function. The callable object / fn should have signature ``scorer(estimator, X, y)``. train : array-like of shape (n_train_samples,) Indices of training samples. test : array-like of shape (n_test_samples,) Indices of test samples. verbose : int The verbosity level. error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit step, which will always raise the error. parameters : dict or None Parameters to be set on the estimator. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. return_train_score : bool, default=False Compute and return score on training set. return_parameters : bool, default=False Return parameters that has been used for the estimator. split_progress : list or tuple, optional, default: None A list or tuple of format (<current_split_id>, <total_num_of_splits>) candidate_progress : list or tuple, optional, default: None A list or tuple of format (<current_candidate_id>, <total_number_of_candidates>) return_n_test_samples : bool, default=False Whether to return the ``n_test_samples`` return_times : bool, default=False Whether to return the fit/score times. return_estimator : bool, default=False Whether to return the fitted estimator. Returns ------- result : dict with the following attributes train_scores : dict of scorer name -> float Score on training set (for all the scorers), returned only if `return_train_score` is `True`. test_scores : dict of scorer name -> float Score on testing set (for all the scorers). n_test_samples : int Number of test samples. fit_time : float Time spent for fitting in seconds. score_time : float Time spent for scoring in seconds. parameters : dict or None The parameters that have been evaluated. estimator : estimator object The fitted estimator. """ progress_msg = "" if verbose > 2: if split_progress is not None: progress_msg = f" {split_progress[0]+1}/{split_progress[1]}" if candidate_progress and verbose > 9: progress_msg += (f"; {candidate_progress[0]+1}/" f"{candidate_progress[1]}") if verbose > 1: if parameters is None: params_msg = '' else: sorted_keys = sorted(parameters) # Ensure deterministic o/p params_msg = (', '.join(f'{k}={parameters[k]}' for k in sorted_keys)) if verbose > 9: start_msg = f"[CV{progress_msg}] START {params_msg}" print(f"{start_msg}{(80 - len(start_msg)) * '.'}") # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() X_train, y_train = _safe_split(estimator, X, y, train) X_test, y_test = _safe_split(estimator, X, y, test, train) # Fit the estimator # Let the custom bag scorer handle fitting of the estimator result = {} try: estimator = _BagScorer_estimator_fit(estimator, X_train, y_train, scorer, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exc()), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: # The estimator is fitted to data correctly # Calculate scoring of estimator using custom scorer fit_time = time.time() - start_time test_scores = _score(estimator, X_test, y_test, scorer) score_time = time.time() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer) if verbose > 1: total_time = score_time + fit_time end_msg = f"[CV{progress_msg}] END " result_msg = params_msg + (";" if params_msg else "") if verbose > 2: if isinstance(test_scores, dict): for scorer_name in sorted(test_scores): result_msg += f" {scorer_name}: (" if return_train_score: result_msg += (f"train=" f"{train_scores[scorer_name]:.3f}, ") result_msg += f"test={test_scores[scorer_name]:.3f})" result_msg += f" total time={logger.short_format_time(total_time)}" # Right align the result_msg end_msg += "." * (80 - len(end_msg) - len(result_msg)) end_msg += result_msg print(end_msg) result["test_scores"] = test_scores if return_train_score: result["train_scores"] = train_scores if return_n_test_samples: # Return the number of bags result["n_test_samples"] = _num_samples(X_test) if return_times: result["fit_time"] = fit_time result["score_time"] = score_time if return_parameters: result["parameters"] = parameters if return_estimator: result["estimator"] = estimator return result
def fit(self, X, y=None, *, groups=None, **fit_params): self.initialize_fitting(X, y) estimator = self.estimator cv = check_cv(self.cv, y, classifier=is_classifier(estimator)) scorers, self.multimetric_ = _check_multimetric_scoring( self.estimator, scoring=self.scoring) if self.multimetric_: if self.refit is not False and ( not isinstance(self.refit, str) or # This will work for both dict / list (tuple) self.refit not in scorers) and not callable(self.refit): raise ValueError("For multi-metric scoring, the parameter " "refit must be set to a scorer key or a " "callable to refit an estimator with the " "best parameter setting on the whole " "data and make the best_* attributes " "available for that metric. If this is " "not needed, refit should be set to " "False explicitly. %r was passed." % self.refit) else: refit_metric = self.refit else: refit_metric = 'score' X, y, groups = indexable(X, y, groups) fit_params = _check_fit_params(X, fit_params) n_splits = cv.get_n_splits(X, y, groups) base_estimator = clone(self.estimator) parallel = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, pre_dispatch=self.pre_dispatch) fit_and_score_kwargs = dict(scorer=scorers, fit_params=fit_params, return_train_score=self.return_train_score, return_n_test_samples=True, return_times=True, return_parameters=False, error_score=self.error_score, verbose=self.verbose) results = {} with parallel: all_candidate_params = [] all_out = [] def evaluate_candidates(candidate_params): candidate_params = list(candidate_params) n_candidates = len(candidate_params) if self.verbose > 0: print("Fitting {0} folds for each of {1} candidates," " totalling {2} fits".format( n_splits, n_candidates, n_candidates * n_splits)) out = parallel( delayed(self._fit_score_and_log)(clone(base_estimator), X, y, train=train, test=test, parameters=parameters, **fit_and_score_kwargs) for parameters, (train, test) in product( candidate_params, cv.split(X, y, groups))) if len(out) < 1: raise ValueError('No fits were performed. ' 'Was the CV iterator empty? ' 'Were there no candidates?') elif len(out) != n_candidates * n_splits: raise ValueError('cv.split and cv.get_n_splits returned ' 'inconsistent results. Expected {} ' 'splits, got {}'.format( n_splits, len(out) // n_candidates)) all_candidate_params.extend(candidate_params) all_out.extend(out) nonlocal results results = self._format_results(all_candidate_params, scorers, n_splits, all_out) return results self._run_search(evaluate_candidates) # For multi-metric evaluation, store the best_index_, best_params_ and # best_score_ iff refit is one of the scorer names # In single metric evaluation, refit_metric is "score" if self.refit or not self.multimetric_: # If callable, refit is expected to return the index of the best # parameter set. if callable(self.refit): self.best_index_ = self.refit(results) if not isinstance(self.best_index_, numbers.Integral): raise TypeError('best_index_ returned is not an integer') if (self.best_index_ < 0 or self.best_index_ >= len(results["params"])): raise IndexError('best_index_ index out of range') else: self.best_index_ = results["rank_test_%s" % refit_metric].argmin() self.best_score_ = results["mean_test_%s" % refit_metric][self.best_index_] self.best_params_ = results["params"][self.best_index_] if self.refit: # we clone again after setting params in case some # of the params are estimators as well. self.best_estimator_ = clone( clone(base_estimator).set_params(**self.best_params_)) refit_start_time = time.time() if y is not None: self.best_estimator_.fit(X, y, **fit_params) else: self.best_estimator_.fit(X, **fit_params) refit_end_time = time.time() self.refit_time_ = refit_end_time - refit_start_time # Store the only scorer not as a dict for single metric evaluation self.scorer_ = scorers if self.multimetric_ else scorers['score'] self.cv_results_ = results self.n_splits_ = n_splits return self
def fit(self, Xs, y=None, *, groups=None, **fit_params): """Run fit with all sets of parameters. Parameters ---------- Xs : array-like of shape (n_samples, n_features) Training vector, where `n_samples` is the number of samples and `n_features` is the number of features. y : array-like of shape (n_samples, n_output) \ or (n_samples,), default=None Target relative to X for classification or regression; None for unsupervised learning. groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`~sklearn.model_selection.GroupKFold`). **fit_params : dict of str -> object Parameters passed to the ``fit`` method of the estimator. Returns ------- self : object Instance of fitted estimator. """ estimator = self.estimator refit_metric = "score" if callable(self.scoring): scorers = self.scoring elif self.scoring is None or isinstance(self.scoring, str): scorers = check_scoring(self.estimator, self.scoring) else: scorers = _check_multimetric_scoring(self.estimator, self.scoring) self._check_refit_for_multimetric(scorers) refit_metric = self.refit fit_params = _check_fit_params(Xs[0], fit_params) cv_orig = check_cv(self.cv, y, classifier=is_classifier(estimator)) n_splits = cv_orig.get_n_splits(Xs[0], y, groups) base_estimator = clone(self.estimator) parallel = Parallel(n_jobs=self.n_jobs, pre_dispatch=self.pre_dispatch) fit_and_score_kwargs = dict( scorer=scorers, fit_params=fit_params, return_train_score=self.return_train_score, return_n_test_samples=True, return_times=True, return_parameters=False, error_score=self.error_score, verbose=self.verbose, ) results = {} with parallel: all_candidate_params = [] all_out = [] all_more_results = defaultdict(list) def evaluate_candidates(candidate_params, cv=None, more_results=None): cv = cv or cv_orig candidate_params = list(candidate_params) n_candidates = len(candidate_params) if self.verbose > 0: print("Fitting {0} folds for each of {1} candidates," " totalling {2} fits".format( n_splits, n_candidates, n_candidates * n_splits)) X_transformed, _, _, n_features = check_Xs( Xs, copy=True, return_dimensions=True) pipeline = Pipeline([ ("splitter", SimpleSplitter(n_features)), ("estimator", clone(base_estimator)), ]) pipeline.fit(np.hstack(Xs)) out = parallel( delayed(_fit_and_score)( pipeline, np.hstack(Xs), y, train=train, test=test, parameters={ f"estimator__{k}": v for k, v in parameters.items() }, split_progress=(split_idx, n_splits), candidate_progress=(cand_idx, n_candidates), **fit_and_score_kwargs, ) for (cand_idx, parameters), (split_idx, (train, test)) in product( enumerate(candidate_params), enumerate(cv.split(Xs[0], y, groups)), )) if len(out) < 1: raise ValueError("No fits were performed. " "Was the CV iterator empty? " "Were there no candidates?") elif len(out) != n_candidates * n_splits: raise ValueError("cv.split and cv.get_n_splits returned " "inconsistent results. Expected {} " "splits, got {}".format( n_splits, len(out) // n_candidates)) # For callable self.scoring, the return type is only know after # calling. If the return type is a dictionary, the error scores # can now be inserted with the correct key. The type checking # of out will be done in `_insert_error_scores`. if callable(self.scoring): _insert_error_scores(out, self.error_score) all_candidate_params.extend(candidate_params) all_out.extend(out) if more_results is not None: for key, value in more_results.items(): all_more_results[key].extend(value) nonlocal results results = self._format_results(all_candidate_params, n_splits, all_out, all_more_results) return results self._run_search(evaluate_candidates) # multimetric is determined here because in the case of a callable # self.scoring the return type is only known after calling first_test_score = all_out[0]["test_scores"] self.multimetric_ = isinstance(first_test_score, dict) # check refit_metric now for a callabe scorer that is multimetric if callable(self.scoring) and self.multimetric_: self._check_refit_for_multimetric(first_test_score) refit_metric = self.refit # For multi-metric evaluation, store the best_index_, best_params_ and # best_score_ iff refit is one of the scorer names # In single metric evaluation, refit_metric is "score" if self.refit or not self.multimetric_: self.best_index_ = self._select_best_index(self.refit, refit_metric, results) if not callable(self.refit): # With a non-custom callable, we can select the best score # based on the best index self.best_score_ = results[f"mean_test_{refit_metric}"][ self.best_index_] self.best_params_ = results["params"][self.best_index_] if self.refit: # we clone again after setting params in case some # of the params are estimators as well. self.best_estimator_ = clone( clone(base_estimator).set_params(**self.best_params_)) refit_start_time = time.time() if y is not None: self.best_estimator_.fit(Xs, y, **fit_params) else: self.best_estimator_.fit(Xs, **fit_params) refit_end_time = time.time() self.refit_time_ = refit_end_time - refit_start_time if hasattr(self.best_estimator_, "feature_names_in_"): self.feature_names_in_ = self.best_estimator_.feature_names_in_ # Store the only scorer not as a dict for single metric evaluation self.scorer_ = scorers self.cv_results_ = results self.n_splits_ = n_splits return self
def fit(self, X, y=None, *, groups=None, **fit_params): """Run fit with all sets of parameters. Parameters ---------- X : array-like of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. y : array-like of shape (n_samples, n_output) \ or (n_samples,), default=None Target relative to X for classification or regression; None for unsupervised learning. groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`~sklearn.model_selection.GroupKFold`). **fit_params : dict of str -> object Parameters passed to the ``fit`` method of the estimator """ estimator = self.estimator refit_metric = "score" if callable(self.scoring): scorers = self.scoring elif self.scoring is None or isinstance(self.scoring, str): scorers = check_scoring(self.estimator, self.scoring) else: scorers = _check_multimetric_scoring(self.estimator, self.scoring) self._check_refit_for_multimetric(scorers) refit_metric = self.refit #X, y, groups = indexable(X, y, groups) # todo debug fit_params = _check_fit_params(X, fit_params) cv_orig = check_cv(self.cv, y, classifier=is_classifier(estimator)) n_splits = cv_orig.get_n_splits(X, y, groups) base_estimator = clone(self.estimator) parallel = Parallel(n_jobs=self.n_jobs, pre_dispatch=self.pre_dispatch) fit_and_score_kwargs = dict(scorer=scorers, fit_params=fit_params, return_train_score=self.return_train_score, return_n_test_samples=True, return_times=True, return_parameters=False, error_score=self.error_score, verbose=self.verbose) results = {} with parallel: all_candidate_params = [] all_out = [] all_more_results = defaultdict(list) def evaluate_candidates(candidate_params, cv=None, more_results=None): cv = cv or cv_orig candidate_params = list(candidate_params) n_candidates = len(candidate_params) if self.verbose > 0: print("Fitting {0} folds for each of {1} candidates," " totalling {2} fits".format( n_splits, n_candidates, n_candidates * n_splits)) if self.online_train_val_split: can = enumerate(candidate_params) spl = enumerate(cv.split(X, None, groups)) lst = [] for (cand_idx, parameters), (split_idx, (train, test)) in product(can, spl): lst.append( delayed(_fit_and_score)( clone(base_estimator), X, y, train=train, test=test, parameters=parameters, online_train_val_split=True, **fit_and_score_kwargs)) out = parallel(lst) else: can = enumerate(candidate_params) spl = enumerate(cv.split(X, y, groups)) lst = [] for (cand_idx, parameters), (split_idx, (train, test)) in product(can, spl): lst.append( delayed(_fit_and_score)( clone(base_estimator), X, y, train=train, test=test, parameters=parameters, split_progress=(split_idx, n_splits), candidate_progress=(cand_idx, n_candidates), online_train_val_split=False, **fit_and_score_kwargs)) out = parallel(lst) # out = parallel(delayed(_fit_and_score)(clone(base_estimator), # X, y, # train=train, test=test, # parameters=parameters, # split_progress=( # split_idx, # n_splits), # candidate_progress=( # cand_idx, # n_candidates), # **fit_and_score_kwargs) # for (cand_idx, parameters), # (split_idx, (train, test)) in product( # enumerate(candidate_params), # enumerate(cv.split(X, y, groups))) # ) if len(out) < 1: raise ValueError('No fits were performed. ' 'Was the CV iterator empty? ' 'Were there no candidates?') elif len(out) != n_candidates * n_splits: raise ValueError('cv.split and cv.get_n_splits returned ' 'inconsistent results. Expected {} ' 'splits, got {}'.format( n_splits, len(out) // n_candidates)) # For callable self.scoring, the return type is only know after # calling. If the return type is a dictionary, the error scores # can now be inserted with the correct key. The type checking # of out will be done in `_insert_error_scores`. if callable(self.scoring): _insert_error_scores(out, self.error_score) all_candidate_params.extend(candidate_params) all_out.extend(out) if more_results is not None: for key, value in more_results.items(): all_more_results[key].extend(value) nonlocal results results = self._format_results(all_candidate_params, n_splits, all_out, all_more_results) return results self._run_search(evaluate_candidates) # multimetric is determined here because in the case of a callable # self.scoring the return type is only known after calling first_test_score = all_out[0]['test_scores'] self.multimetric_ = isinstance(first_test_score, dict) # check refit_metric now for a callabe scorer that is multimetric if callable(self.scoring) and self.multimetric_: self._check_refit_for_multimetric(first_test_score) refit_metric = self.refit # For multi-metric evaluation, store the best_index_, best_params_ and # best_score_ iff refit is one of the scorer names # In single metric evaluation, refit_metric is "score" if self.refit or not self.multimetric_: # If callable, refit is expected to return the index of the best # parameter set. if callable(self.refit): self.best_index_ = self.refit(results) if not isinstance(self.best_index_, np.numbers.Integral): raise TypeError('best_index_ returned is not an integer') if (self.best_index_ < 0 or self.best_index_ >= len(results["params"])): raise IndexError('best_index_ index out of range') else: self.best_index_ = results["rank_test_%s" % refit_metric].argmin() self.best_score_ = results["mean_test_%s" % refit_metric][self.best_index_] self.best_params_ = results["params"][self.best_index_] if self.refit: # we clone again after setting params in case some # of the params are estimators as well. self.best_estimator_ = clone( clone(base_estimator).set_params(**self.best_params_)) refit_start_time = time.time() if isinstance(self.best_estimator_, Pipeline): self.best_estimator_.train() # todo set train intervall to whole dataset if y is not None: self.best_estimator_.fit(X, y, **fit_params) else: self.best_estimator_.fit(X, **fit_params) if isinstance(self.best_estimator_, Pipeline): self.best_estimator_.prod() refit_end_time = time.time() self.refit_time_ = refit_end_time - refit_start_time # Store the only scorer not as a dict for single metric evaluation self.scorer_ = scorers self.cv_results_ = results self.n_splits_ = n_splits return self
def _fit_and_score_weighted(estimator, x_data, y_data, scorer, train, test, verbose, parameters, fit_params, error_score=np.nan, sample_weights=None): """Expand :func:`sklearn.model_selection._validation._fit_and_score`.""" if verbose > 1: if parameters is None: msg = '' else: msg = '%s' % (', '.join('%s=%s' % (k, v) for k, v in parameters.items())) print("[CV] %s %s" % (msg, (64 - len(msg)) * '.')) # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(x_data, fit_params, train) if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for (key, val) in parameters.items(): cloned_parameters[key] = clone(val, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() x_train, y_train = _safe_split(estimator, x_data, y_data, train) x_test, y_test = _safe_split(estimator, x_data, y_data, test, train) if sample_weights is not None: sample_weights_test = sample_weights[test] else: sample_weights_test = None try: if y_train is None: estimator.fit(x_train, **fit_params) else: estimator.fit(x_train, y_train, **fit_params) except Exception as exc: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise if isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} else: test_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exception_only(type(exc), exc)[0]), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: fit_time = time.time() - start_time test_scores = _score_weighted(estimator, x_test, y_test, scorer, sample_weights=sample_weights_test) score_time = time.time() - start_time - fit_time if verbose > 2: if isinstance(test_scores, dict): for scorer_name in sorted(test_scores): msg += ", %s=" % scorer_name msg += "%.3f" % test_scores[scorer_name] else: msg += ", score=" msg += "%.3f" % test_scores if verbose > 1: total_time = score_time + fit_time print(_message_with_time('CV', msg, total_time)) return [test_scores]
def _compute_one_fold( fold_index, train, test, multi_output_proba, all_classes, classes, estimator, X, y, groups, scorers, verbose, fit_params, return_predict, method, no_scoring, ): if verbose: print("cv %d started\n" % fold_index) ### Clone the estimator ### cloned_estimator = sklearn.base.clone(estimator) ### split train test ### X_train, y_train = sklearn.model_selection._validation._safe_split( estimator, X, y, train) if groups is not None: groups_train, _ = sklearn.model_selection._validation._safe_split( estimator, groups, None, train) else: groups_train = None X_test, y_test = sklearn.model_selection._validation._safe_split( estimator, X, y, test, train) if groups is not None: groups_test, _ = sklearn.model_selection._validation._safe_split( estimator, groups, None, test, train) else: groups_test = None if hasattr(X_test, "index"): index_test = X_test.index else: index_test = test fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) # Try to subset the fit_params if that is possible, Ex : 'sample_weight=np.array(....)' should be subsetted but not 'epochs=10' start_fit = time() ### Fit estimator ### if y_train is None: if groups_train is not None and function_has_named_argument( cloned_estimator.fit, "groups"): cloned_estimator.fit(X_train, groups=groups_train, **fit_params) else: cloned_estimator.fit(X_train, **fit_params) else: if groups_train is not None and function_has_named_argument( cloned_estimator.fit, "groups"): cloned_estimator.fit(X_train, y_train, groups=groups_train, **fit_params) else: cloned_estimator.fit(X_train, y_train, **fit_params) fit_time = time() - start_fit result_predict = None if return_predict: func = getattr(cloned_estimator, method) predictions = func(X_test) ## re-alignement with class ## if method in ("predict_proba", "predict_log_proba", "decision_function"): def _align_predict(predictions, classes, cloned_estimator_classes_): float_min = np.finfo(predictions.dtype).min default_values = { "decision_function": float_min, "predict_log_proba": float_min, "predict_proba": 0 } predictions_for_all_classes = pd.DataFrame( default_values[method], index=index_test, columns=classes) for j, c in enumerate(cloned_estimator_classes_): predictions_for_all_classes[c] = predictions[:, j] return predictions_for_all_classes if multi_output_proba: predictions = [ _align_predict(p, c, cloned_c) for p, c, cloned_c in zip( predictions, all_classes, cloned_estimator.classes_) ] else: predictions = _align_predict(predictions, classes, cloned_estimator.classes_) result_predict = (predictions, test) result = OrderedDict() ### Score test ### test_scores_dictionary = None if not no_scoring: start_score = time() test_scores_dictionary = _score_with_group(cloned_estimator, X_test, y_test, groups_test, scorer=scorers, is_multimetric=True) # Here : scorers is a dictionary of scorers, hence is_multimetric = True score_time = time() - start_score ### Score train ### train_scores_dictionary = _score_with_group(cloned_estimator, X_train, y_train, groups_train, scorer=scorers, is_multimetric=True) ### Put everything into a dictionnary ### for k, v in test_scores_dictionary.items(): result["test_%s" % k] = v for k, v in train_scores_dictionary.items(): result["train_%s" % k] = v result["fit_time"] = fit_time if not no_scoring: result["score_time"] = score_time result[ "n_test_samples"] = sklearn.model_selection._validation._num_samples( X_test) result["fold_nb"] = fold_index return result, result_predict, test_scores_dictionary
def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params, method): """Fit estimator and predict values for a given dataset split. Read more in the :ref:`User Guide <cross_validation>`. Parameters ---------- estimator : estimator object implementing 'fit' and 'predict' The object to use to fit the data. X : array-like of shape (n_samples, n_features) The data to fit. .. versionchanged:: 0.20 X is only required to be an object with finite length or shape now y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The target variable to try to predict in the case of supervised learning. train : array-like of shape (n_train_samples,) Indices of training samples. test : array-like of shape (n_test_samples,) Indices of test samples. verbose : int The verbosity level. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. method : str Invokes the passed method name of the passed estimator. Returns ------- predictions : sequence Result of calling 'estimator.method' test : array-like This is the value of the test parameter """ # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) X_train, y_train = _safe_split(estimator, X, y, train) X_test, _ = _safe_split(estimator, X, y, test, train) if y_train is None: estimator.fit(X_train, **fit_params) else: estimator.fit(X_train, y_train, **fit_params) func = getattr(estimator, method) assert not np.any(np.isinf(X_test)), X_test assert not np.any(np.isnan(X_test)), X_test predictions = func(X_test) encode = method in [ 'decision_function', 'predict_proba', 'predict_log_proba' ] and y is not None if encode: if isinstance(predictions, list): predictions = [ _enforce_prediction_order(estimator.classes_[i_label], predictions[i_label], n_classes=len(set(y[:, i_label])), method=method) for i_label in range(len(predictions)) ] else: # A 2D y array should be a binary label indicator matrix n_classes = len(set(y)) if y.ndim == 1 else y.shape[1] predictions = _enforce_prediction_order(estimator.classes_, predictions, n_classes, method) return predictions, test, estimator
def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, return_estimator=False, error_score=np.nan): if verbose > 1: if parameters is None: msg = '' else: msg = '%s' % (', '.join('%s=%s' % (k, v) for k, v in parameters.items())) print("[CV] %s %s" % (msg, (64 - len(msg)) * '.')) # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) train_scores = {} if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.time() X_train, y_train = _safe_split(estimator, X, y, train) #print(X_train.shape) X_train = pd.concat([ X_train, pd.concat([X_train.iloc[:, 24:], X_train.iloc[:, :24]], axis=1) ]) #X_train.iloc[:, :33], X_train.iloc[:, 33:]], axis=1) y_train = np.concatenate([y_train, y_train]) #print(X_train.shape) X_test, y_test = _safe_split(estimator, X, y, test, train) try: if y_train is None: estimator.fit(X_train, **fit_params) else: estimator.fit(X_train, y_train, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exception_only(type(e), e)[0]), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: fit_time = time.time() - start_time test_scores = _score(estimator, X_test, y_test, scorer) score_time = time.time() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer) if verbose > 2: if isinstance(test_scores, dict): for scorer_name in sorted(test_scores): msg += ", %s=" % scorer_name if return_train_score: msg += "(train=%.3f," % train_scores[scorer_name] msg += " test=%.3f)" % test_scores[scorer_name] else: msg += "%.3f" % test_scores[scorer_name] else: msg += ", score=" msg += ("%.3f" % test_scores if not return_train_score else "(train=%.3f, test=%.3f)" % (train_scores, test_scores)) if verbose > 1: total_time = score_time + fit_time print(_message_with_time('CV', msg, total_time)) ret = [train_scores, test_scores] if return_train_score else [test_scores] if return_n_test_samples: ret.append(_num_samples(X_test)) if return_times: ret.extend([fit_time, score_time]) if return_parameters: ret.append(parameters) if return_estimator: ret.append(estimator) return ret
def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=True, return_estimator=False, error_score=np.nan): if verbose > 1: if parameters is None: msg = '' else: msg = '%s' % (', '.join('%s=%s' % (k, v) for k, v in parameters.items())) print("[CV] %s %s" % (msg, (64 - len(msg)) * '.')) # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = _check_fit_params(X, fit_params, train) train_scores = {} if parameters is not None: # clone after setting parameters in case any parameters # are estimators (like pipeline steps) # because pipeline doesn't clone steps in fit cloned_parameters = {} for k, v in parameters.items(): cloned_parameters[k] = clone(v, safe=False) estimator = estimator.set_params(**cloned_parameters) start_time = time.perf_counter() X_train, y_train = _safe_split(estimator, X, y, train) X_test, y_test = _safe_split(estimator, X, y, test, train) try: if y_train is None: estimator.fit(X_train, **fit_params) else: estimator.fit(X_train, y_train, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.perf_counter() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if isinstance(scorer, dict): test_scores = {name: error_score for name in scorer} if return_train_score: train_scores = test_scores.copy() else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % (error_score, format_exception_only(type(e), e)[0]), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: fit_time = time.perf_counter() - start_time test_scores = _score(estimator, X_test, y_test, scorer) score_time = time.perf_counter() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer) if verbose > 2: if isinstance(test_scores, dict): for scorer_name in sorted(test_scores): msg += ", %s=" % scorer_name if return_train_score: msg += "(train=%.3f," % train_scores[scorer_name] msg += " test=%.3f)" % test_scores[scorer_name] else: msg += "%.3f" % test_scores[scorer_name] else: msg += ", score=" msg += ("%.3f" % test_scores if not return_train_score else "(train=%.3f, test=%.3f)" % (train_scores, test_scores)) total_time = score_time + fit_time if verbose > 1: print(_message_with_time('CV', msg, total_time)) ret = [train_scores, test_scores] if return_train_score else [test_scores] p = pickle.dumps(estimator) mem = sys.getsizeof(p) #in bytes #TODO переделать без костылей #for numpy if (isinstance(X_test, np.ndarray)): inf_example = X_test[0].reshape(1, -1) #for DataFrame if (isinstance(X_test, pd.DataFrame)): inf_example = X_test.iloc[0].to_numpy().reshape(1, -1) #Reshape your data either using array.reshape(-1, 1) #if your data has a single feature or array.reshape(1, -1) if it contains #a single sample. inference_time_start = time.perf_counter() s = estimator.predict(inf_example) inference_time_end = time.perf_counter() inference_time = inference_time_end - inference_time_start if return_n_test_samples: ret.append(_num_samples(X_test)) if return_times: ret.extend([fit_time, score_time, inference_time, mem, total_time]) if return_parameters: ret.append(parameters) if return_estimator: ret.append(estimator) return ret