def _unit_test_params(cls):
yield {
"models": [
compose.Pipeline(
preprocessing.StandardScaler(),
linear_model.LinearRegression(optimizer=optim.SGD(
lr=1e-2)),
),
compose.Pipeline(
preprocessing.StandardScaler(),
linear_model.LinearRegression(optimizer=optim.SGD(
lr=1e-1)),
),
],
"metric":
metrics.MAE(),
}
yield {
"models": [
compose.Pipeline(
preprocessing.StandardScaler(),
linear_model.LinearRegression(optimizer=optim.SGD(lr=lr)),
) for lr in [1e-4, 1e-3, 1e-2, 1e-1]
],
"metric":
metrics.MAE(),
}
def test_compose():
metrics.MAE() + metrics.MSE()
metrics.Accuracy() + metrics.LogLoss()
with pytest.raises(ValueError):
_ = metrics.MSE() + metrics.LogLoss()
with pytest.raises(ValueError):
_ = metrics.MSE() + metrics.MAE() + metrics.LogLoss()
def reset(self):
self.mae = metrics.Rolling(metrics.MAE(), window_size=self.window_size)
self.mse = metrics.Rolling(metrics.MSE(), window_size=self.window_size)
self.r2 = metrics.Rolling(metrics.R2(), window_size=self.window_size)
self.sample_count = 0
self.last_true_label = None
self.last_prediction = None
def evaluate_model(model):
metric = metrics.Rolling(metrics.MAE(), 12)
# dates = []
y_trues = []
y_preds = []
for x, y in timeonly_training_dataset(target_data):
# Obtain the prior prediction and update the model in one go
y_pred = model.predict_one(x)
model.learn_one(x, y)
# Update the error metric
metric.update(y, y_pred)
# Store the true value and the prediction
# dates.append(x['secs_elapsed'])
y_trues.append(y)
y_preds.append(y_pred)
# Plot the results
fig, ax = plt.subplots(figsize=(10, 6))
ax.grid(alpha=0.75)
ax.plot(y_trues, lw=3, color='#2ecc71', alpha=0.8, label='Ground truth')
ax.plot(y_preds, lw=3, color='#e74c3c', alpha=0.8, label='Prediction')
ax.legend()
ax.set_title(metric)
plt.show()
def __init__(self):
super().__init__()
self.mae = metrics.MAE()
self.mse = metrics.MSE()
self.r2 = metrics.R2()
self.last_true_label = None
self.last_prediction = None
def __init__(self, window_size=200):
super().__init__()
self.window_size = window_size
self.mae = metrics.Rolling(metrics.MAE(), window_size=self.window_size)
self.mse = metrics.Rolling(metrics.MSE(), window_size=self.window_size)
self.r2 = metrics.Rolling(metrics.R2(), window_size=self.window_size)
self.sample_count = 0
self.last_true_label = None
self.last_prediction = None
def __init__(self, models, epsilon=0.1, decay=0.0, metric=None, seed=None):
if metric is None:
metric = metrics.MAE()
super().__init__(
models=models,
metric=metric,
solver=EpsilonGreedy(
bandit=Bandit(n_arms=len(models), metric=metric,),
epsilon=epsilon,
decay=decay,
seed=seed,
),
)
def _unit_test_params(cls):
return {
"models": [
compose.Pipeline(
preprocessing.StandardScaler(),
linear_model.LinearRegression(optimizer=optim.SGD(
lr=0.01)),
),
compose.Pipeline(
preprocessing.StandardScaler(),
linear_model.LinearRegression(optimizer=optim.SGD(lr=0.1)),
),
],
"metric":
metrics.MAE(),
}
def __init__(self, step, name):
self.name = name
self.optimizer = SynchronousSGD(0.01, name, None)
self.model = compose.Pipeline(
preprocessing.StandardScaler(),
linear_model.LogisticRegression(self.optimizer))
self.metrics = [
metrics.Accuracy(),
metrics.MAE(),
metrics.RMSE(),
metrics.Precision(),
metrics.Recall()
]
self.count = 0
if step is None:
self.step = 50
else:
self.step = int(step)
def elevate_model(model):
metric = metrics.Rolling(metrics.MAE(), 12)
for x, y in datasets.AirlinePassengers():
y_trues.append(y)
dates.append(x['month'])
iter = 0
for x, y in datasets.AirlinePassengers():
#Obtain the prior prediction and update the model in one go
y_pred = model.predict_one(x)
model.learn_one(x, y)
y_pred_post = model.predict_one(x)
#Update the error metric
metric.update(y, y_pred)
#Store the true value and the prediction
dates_pred.append(x['month'])
y_preds.append(y_pred)
iter += 1
fig, ax = plt.subplots(figsize=(10, 6))
ax.grid(alpha=0.5)
ax.plot(dates,
y_trues,
lw=3,
color='#2ecc71',
alpha=0.8,
label='Ground Truth')
ax.plot(dates,
y_preds,
lw=3,
color='#e74c3c',
alpha=0.8,
label='Prediction')
ax.legend()
ax.set_title("Airline Passenger Example")
plt.show()
print(metric)
def __init__(
self,
models,
metric=None,
delta=1,
burn_in=100,
seed: int = None,
):
if metric is None:
metric = metrics.MAE()
super().__init__(
models=models,
metric=metric,
policy=UCB(
delta=delta,
burn_in=burn_in,
seed=seed,
),
)
def __init__(
self,
models,
metric=None,
epsilon=0.1,
decay=0.0,
burn_in=100,
seed: int = None,
):
if metric is None:
metric = metrics.MAE()
super().__init__(
models=models,
metric=metric,
policy=EpsilonGreedy(
epsilon=epsilon,
decay=decay,
burn_in=burn_in,
seed=seed,
),
)
def __init__(
self,
models: typing.List[base.Estimator],
metric: metrics.RegressionMetric = None,
delta: float = None,
explore_each_arm: int = 1,
start_after: int = 20,
seed: int = None,
):
if metric is None:
metric = metrics.MAE()
super().__init__(
models=models,
metric=metric,
delta=delta,
explore_each_arm=explore_each_arm,
start_after=start_after,
seed=seed,
)
def test_forecasts_at_each_step():
dataset = datasets.AirlinePassengers()
model = MeanForecaster()
metric = metrics.MAE()
horizon = 12
grace_period = 1
steps = _evaluate(dataset, model, metric, horizon, grace_period)
y_pred, _ = next(steps)
assert y_pred == [112] * horizon
y_pred, _ = next(steps)
assert y_pred == [(112 + 118) / 2] * horizon
y_pred, _ = next(steps)
assert y_pred == [(112 + 118 + 132) / 3] * horizon
y_pred, _ = next(steps)
assert y_pred == [(112 + 118 + 132 + 129) / 4] * horizon
n_steps = sum(
1 for _ in _evaluate(dataset, model, metric, horizon, grace_period))
assert n_steps == dataset.n_samples - horizon - grace_period
def __init__(
self,
models: typing.List[base.Estimator],
metric: metrics.RegressionMetric = None,
epsilon: float = 0.1,
epsilon_decay: float = None,
explore_each_arm: int = 3,
start_after: int = 20,
seed: int = None,
):
if metric is None:
metric = metrics.MAE()
super().__init__(
models=models,
metric=metric,
epsilon=epsilon,
epsilon_decay=epsilon_decay,
explore_each_arm=explore_each_arm,
start_after=start_after,
seed=seed,
)
def get_metric(period):
return metrics.Rolling(metrics.MAE(), period)
(metrics.MicroRecall(), partial(sk_metrics.recall_score, average='micro')),
(metrics.WeightedRecall(),
partial(sk_metrics.recall_score, average='weighted')),
(metrics.FBeta(beta=.5), partial(sk_metrics.fbeta_score, beta=.5)),
(metrics.MacroFBeta(beta=.5),
partial(sk_metrics.fbeta_score, beta=.5, average='macro')),
(metrics.MicroFBeta(beta=.5),
partial(sk_metrics.fbeta_score, beta=.5, average='micro')),
(metrics.WeightedFBeta(beta=.5),
partial(sk_metrics.fbeta_score, beta=.5, average='weighted')),
(metrics.F1(), sk_metrics.f1_score),
(metrics.MacroF1(), partial(sk_metrics.f1_score, average='macro')),
(metrics.MicroF1(), partial(sk_metrics.f1_score, average='micro')),
(metrics.WeightedF1(), partial(sk_metrics.f1_score, average='weighted')),
(metrics.MCC(), sk_metrics.matthews_corrcoef),
(metrics.MAE(), sk_metrics.mean_absolute_error),
(metrics.MSE(), sk_metrics.mean_squared_error),
]
@pytest.mark.parametrize('metric, sk_metric', [
pytest.param(metric, sk_metric, id=f'{metric.__class__.__name__}')
for metric, sk_metric in TEST_CASES
])
@pytest.mark.filterwarnings('ignore::RuntimeWarning')
@pytest.mark.filterwarnings(
'ignore::sklearn.metrics.classification.UndefinedMetricWarning')
def test_metric(metric, sk_metric):
# Check str works
str(metric)
# construct a model pipeline
model = compose.Select('clouds', 'humidity', 'pressure', 'temperature', 'wind')
# To extract feature (bike number by hour)
model += (
get_hour
| feature_extraction.TargetAgg(by=['station', 'hour'], how=stats.Mean()) +
feature_extraction.TargetAgg(by='station', how=stats.EWMean(
0.5)) # aggregate feature (station and time)
)
model |= preprocessing.StandardScaler()
model |= linear_model.LinearRegression()
evaluate.progressive_val_score(dataset=datasets.Bikes(),
model=model,
metric=metrics.MAE(),
moment='moment',
delay=dt.timedelta(minutes=30),
print_every=20_000)
for x, y, in itertools.islice(X_y, 10000):
y_pred = model.predict_one(x)
model.learn_one(x, y)
x, y = next(X_y)
print(model.debug_one(x))
metrics = metrics.MAE()
def reset(self):
self.mae = metrics.MAE()
self.mse = metrics.MSE()
self.r2 = metrics.R2()
self.last_true_label = None
self.last_prediction = None
# model = (
# preprocessing.StandardScaler() |
# RadiusNeighborsRegressor(max_size=1000, r=1, aggregation='distance',
# delta=0.1, seed=42, k=4)
# )
model = (preprocessing.StandardScaler()
| KNNRegressor(n_neighbors=3,
window_size=1000,
r=1,
c=10,
aggregation_method='mean',
delta=0.1,
seed=42,
k=4))
print(f'L={model["KNNRegressor"]._buffer.L}')
print(f'P1={model["KNNRegressor"]._buffer.P1}')
print(f'P2={model["KNNRegressor"]._buffer.P2}')
# exit()
metric = metrics.MAE()
evaluate.progressive_val_score(dataset,
model,
metric,
print_every=100,
show_time=True,
show_memory=True)
