def calculate(self, Y_hat, Y, average='micro', weights=None):
Y_hat, Y = check_XY(X=Y_hat, Y=Y)
return metrics.fbeta_score(Y,
Y_hat,
self.beta,
average=average,
sample_weight=weights)
def calculate(self, Y_hat, Y, average='macro', weights=None):
Y_hat, Y = check_XY(X=Y_hat, Y=Y)
return metrics.roc_auc_score(Y,
Y_hat,
average=average,
multi_class=self.multi_class,
sample_weight=weights)
def _cross_validate_timeseries(estimator, X, Y, cv, repeat, metric,
random_state, shuffle, verbose):
"""
Conduct a k-fold cross validation on a TimeSeriesClassifier.
Parameters
----------
estimator : TimeSeriesClassifier
Estimator to use.
X : list of array-like, shape=(n_series, n_samples, n_features)
Data to use for cross-validation. Series can be of different lengths.
Y : list of array-like, shape=(n_series, n_samples,)
Target labels to use for cross-validation.
cv : int
Number of folds to use in k-fold cross-validation
repeat : int
Number of times to repeat cross-validation to average
the results.
metric : Metric
Metric to use for scoring.
random_state : RandomState
RandomState for shuffling.
shuffle : bool
Determine if cross-validation should shuffle the data.
verbose : int
Determines the verbosity of cross-validation.
Higher verbose levels result in more output logged.
Returns
-------
score : float
The average score across the cross-validation.
Y_hat : list of ndarray, shape=(n_series, n_samples, n_classes)
The cross-validated predictions.
"""
e_verbose = estimator.verbose
suppressed_verbose = verbose - 1 if verbose >= 1 else 0
estimator.set_verbose(suppressed_verbose)
for x, y in zip(X, Y):
check_XY(X=x, Y=y)
try:
X_, Y_ = np.concatenate(X), np.concatenate(Y)
except:
raise ValueError("Inputs have different number of features")
for i in range(len(Y)):
if Y[i].ndim == 1: Y[i] = Y[i].reshape(-1, 1)
Y_hat = [np.zeros((len(y), len(np.unique(y)))) for y in Y]
lens = [len(y) for y in Y]
for r in range(repeat):
if verbose > 0: print("Fitting on repeat", r + 1)
X_s, Y_s, i_s = sk_shuffle(X,
Y,
np.arange(len(X), dtype=int),
random_state=random_state)
X_folds, Y_folds, i_folds = (np.array_split(
np.array(X_s, dtype=object),
cv), np.array_split(np.array(Y_s, dtype=object), cv),
np.array_split(
np.array(i_s, dtype=object), cv))
if verbose == 1: folds = trange(cv)
else: folds = range(cv)
for f in folds:
if verbose > 1: print("Fitting on fold", f + 1)
x_ = np.concatenate(
X_folds[:-1]) if len(X_folds) > 2 else X_folds[0]
y_ = np.concatenate(
Y_folds[:-1]) if len(Y_folds) > 2 else Y_folds[0]
e = deepcopy(estimator)
e.fit(x_, y_)
if verbose > 1: print("Predicting for fold", f + 1)
Y_hat_ = e.predict_proba(X_folds[-1])
if verbose > 2:
scores_, lens_ = [], [len(y_f) for y_f in Y_folds[-1]]
for y_hat, y in zip(Y_hat_, Y_folds[-1]):
p = np.argmax(y_hat, axis=1)
scores_.append(metric.score(p, y))
score_ = np.average(scores_, weights=lens_)
print("Score:", score_)
for i in range(len(Y_hat_)):
Y_hat[i_folds[-1][i]] += Y_hat_[i] / repeat
X_folds = X_folds[-1:] + X_folds[:-1]
Y_folds = Y_folds[-1:] + Y_folds[:-1]
i_folds = i_folds[-1:] + i_folds[:-1]
estimator.set_verbose(e_verbose)
scores = []
for y_hat, y in zip(Y_hat, Y):
p = np.argmax(y_hat, axis=1)
scores.append(metric.score(p, y))
score = np.average(scores, weights=lens)
return score, Y_hat
def _cross_validate_classic(estimator, X, Y, cv, repeat, metric, random_state,
shuffle, verbose):
"""
Conduct a k-fold cross validation on a Classifier.
Parameters
----------
estimator : Classifier
Estimator to use.
X : array-like, shape=(n_samples, n_features)
Data to use for cross-validation.
Y : array-like, shape=(n_samples,)
Target labels to use for cross-validation.
cv : int
Number of folds to use in k-fold cross-validation
repeat : int
Number of times to repeat cross-validation to average
the results.
metric : Metric
Metric to use for scoring.
random_state : RandomState
RandomState for shuffling.
shuffle : bool
Determine if cross-validation should shuffle the data.
verbose : int
Determines the verbosity of cross-validation.
Higher verbose levels result in more output logged.
Returns
-------
score : float
The average score across the cross-validation.
Y_hat : ndarray, shape=(n_samples, n_classes)
The cross-validated predictions.
"""
e_verbose = estimator.verbose
suppressed_verbose = verbose - 1 if verbose >= 1 else 0
estimator.set_verbose(suppressed_verbose)
X, Y = check_XY(X=X, Y=Y)
if Y.ndim == 1: Y = Y.reshape(-1, 1)
Y_hat = np.zeros((len(Y), len(np.unique(Y))))
for r in range(repeat):
if verbose > 0: print("Fitting on repeat", r + 1)
x_y = np.concatenate((X, Y, np.arange(len(X)).reshape(-1, 1)), axis=1)
if shuffle: random_state.shuffle(x_y)
X_s, Y_s, i_s = x_y[:, :X.shape[-1]], x_y[:, X.shape[-1]:-1], x_y[:,
-1]
X_folds, Y_folds, i_folds = (np.array_split(X_s, cv),
np.array_split(Y_s, cv),
np.array_split(i_s, cv))
if verbose == 1: folds = trange(cv)
else: folds = range(cv)
for f in folds:
if verbose > 1: print("Fitting on fold", f + 1)
x_ = np.concatenate(
X_folds[:-1]) if len(X_folds) > 2 else X_folds[0]
y_ = np.concatenate(
Y_folds[:-1]) if len(Y_folds) > 2 else Y_folds[0]
e = deepcopy(estimator)
e.fit(x_, y_)
if verbose > 1: print("Predicting for fold", f + 1)
Y_hat_ = e.predict_proba(X_folds[-1])
if verbose > 2:
score_ = metric.score(np.argmax(Y_hat_, axis=1), Y_folds[-1])
print("Score:", score_)
Y_hat[i_folds[-1].astype(int)] += Y_hat_ / repeat
X_folds = X_folds[-1:] + X_folds[:-1]
Y_folds = Y_folds[-1:] + Y_folds[:-1]
i_folds = i_folds[-1:] + i_folds[:-1]
estimator.set_verbose(e_verbose)
p = np.argmax(Y_hat, axis=1)
return metric.score(p, Y), Y_hat
def calculate(self, Y_hat, Y, weights=None):
Y_hat, Y = check_XY(X=Y_hat, Y=Y)
return metrics.accuracy_score(Y, Y_hat, sample_weight=weights)