def _unit_test_params(cls): yield { "models": [ compose.Pipeline( preprocessing.StandardScaler(), linear_model.LinearRegression(optimizer=optim.SGD( lr=1e-2)), ), compose.Pipeline( preprocessing.StandardScaler(), linear_model.LinearRegression(optimizer=optim.SGD( lr=1e-1)), ), ], "metric": metrics.MAE(), } yield { "models": [ compose.Pipeline( preprocessing.StandardScaler(), linear_model.LinearRegression(optimizer=optim.SGD(lr=lr)), ) for lr in [1e-4, 1e-3, 1e-2, 1e-1] ], "metric": metrics.MAE(), }
def test_compose(): metrics.MAE() + metrics.MSE() metrics.Accuracy() + metrics.LogLoss() with pytest.raises(ValueError): _ = metrics.MSE() + metrics.LogLoss() with pytest.raises(ValueError): _ = metrics.MSE() + metrics.MAE() + metrics.LogLoss()
def reset(self): self.mae = metrics.Rolling(metrics.MAE(), window_size=self.window_size) self.mse = metrics.Rolling(metrics.MSE(), window_size=self.window_size) self.r2 = metrics.Rolling(metrics.R2(), window_size=self.window_size) self.sample_count = 0 self.last_true_label = None self.last_prediction = None
def evaluate_model(model): metric = metrics.Rolling(metrics.MAE(), 12) # dates = [] y_trues = [] y_preds = [] for x, y in timeonly_training_dataset(target_data): # Obtain the prior prediction and update the model in one go y_pred = model.predict_one(x) model.learn_one(x, y) # Update the error metric metric.update(y, y_pred) # Store the true value and the prediction # dates.append(x['secs_elapsed']) y_trues.append(y) y_preds.append(y_pred) # Plot the results fig, ax = plt.subplots(figsize=(10, 6)) ax.grid(alpha=0.75) ax.plot(y_trues, lw=3, color='#2ecc71', alpha=0.8, label='Ground truth') ax.plot(y_preds, lw=3, color='#e74c3c', alpha=0.8, label='Prediction') ax.legend() ax.set_title(metric) plt.show()
def __init__(self): super().__init__() self.mae = metrics.MAE() self.mse = metrics.MSE() self.r2 = metrics.R2() self.last_true_label = None self.last_prediction = None
def __init__(self, window_size=200): super().__init__() self.window_size = window_size self.mae = metrics.Rolling(metrics.MAE(), window_size=self.window_size) self.mse = metrics.Rolling(metrics.MSE(), window_size=self.window_size) self.r2 = metrics.Rolling(metrics.R2(), window_size=self.window_size) self.sample_count = 0 self.last_true_label = None self.last_prediction = None
def __init__(self, models, epsilon=0.1, decay=0.0, metric=None, seed=None): if metric is None: metric = metrics.MAE() super().__init__( models=models, metric=metric, solver=EpsilonGreedy( bandit=Bandit(n_arms=len(models), metric=metric,), epsilon=epsilon, decay=decay, seed=seed, ), )
def _unit_test_params(cls): return { "models": [ compose.Pipeline( preprocessing.StandardScaler(), linear_model.LinearRegression(optimizer=optim.SGD( lr=0.01)), ), compose.Pipeline( preprocessing.StandardScaler(), linear_model.LinearRegression(optimizer=optim.SGD(lr=0.1)), ), ], "metric": metrics.MAE(), }
def __init__(self, step, name): self.name = name self.optimizer = SynchronousSGD(0.01, name, None) self.model = compose.Pipeline( preprocessing.StandardScaler(), linear_model.LogisticRegression(self.optimizer)) self.metrics = [ metrics.Accuracy(), metrics.MAE(), metrics.RMSE(), metrics.Precision(), metrics.Recall() ] self.count = 0 if step is None: self.step = 50 else: self.step = int(step)
def elevate_model(model): metric = metrics.Rolling(metrics.MAE(), 12) for x, y in datasets.AirlinePassengers(): y_trues.append(y) dates.append(x['month']) iter = 0 for x, y in datasets.AirlinePassengers(): #Obtain the prior prediction and update the model in one go y_pred = model.predict_one(x) model.learn_one(x, y) y_pred_post = model.predict_one(x) #Update the error metric metric.update(y, y_pred) #Store the true value and the prediction dates_pred.append(x['month']) y_preds.append(y_pred) iter += 1 fig, ax = plt.subplots(figsize=(10, 6)) ax.grid(alpha=0.5) ax.plot(dates, y_trues, lw=3, color='#2ecc71', alpha=0.8, label='Ground Truth') ax.plot(dates, y_preds, lw=3, color='#e74c3c', alpha=0.8, label='Prediction') ax.legend() ax.set_title("Airline Passenger Example") plt.show() print(metric)
def __init__( self, models, metric=None, delta=1, burn_in=100, seed: int = None, ): if metric is None: metric = metrics.MAE() super().__init__( models=models, metric=metric, policy=UCB( delta=delta, burn_in=burn_in, seed=seed, ), )
def __init__( self, models, metric=None, epsilon=0.1, decay=0.0, burn_in=100, seed: int = None, ): if metric is None: metric = metrics.MAE() super().__init__( models=models, metric=metric, policy=EpsilonGreedy( epsilon=epsilon, decay=decay, burn_in=burn_in, seed=seed, ), )
def __init__( self, models: typing.List[base.Estimator], metric: metrics.RegressionMetric = None, delta: float = None, explore_each_arm: int = 1, start_after: int = 20, seed: int = None, ): if metric is None: metric = metrics.MAE() super().__init__( models=models, metric=metric, delta=delta, explore_each_arm=explore_each_arm, start_after=start_after, seed=seed, )
def test_forecasts_at_each_step(): dataset = datasets.AirlinePassengers() model = MeanForecaster() metric = metrics.MAE() horizon = 12 grace_period = 1 steps = _evaluate(dataset, model, metric, horizon, grace_period) y_pred, _ = next(steps) assert y_pred == [112] * horizon y_pred, _ = next(steps) assert y_pred == [(112 + 118) / 2] * horizon y_pred, _ = next(steps) assert y_pred == [(112 + 118 + 132) / 3] * horizon y_pred, _ = next(steps) assert y_pred == [(112 + 118 + 132 + 129) / 4] * horizon n_steps = sum( 1 for _ in _evaluate(dataset, model, metric, horizon, grace_period)) assert n_steps == dataset.n_samples - horizon - grace_period
def __init__( self, models: typing.List[base.Estimator], metric: metrics.RegressionMetric = None, epsilon: float = 0.1, epsilon_decay: float = None, explore_each_arm: int = 3, start_after: int = 20, seed: int = None, ): if metric is None: metric = metrics.MAE() super().__init__( models=models, metric=metric, epsilon=epsilon, epsilon_decay=epsilon_decay, explore_each_arm=explore_each_arm, start_after=start_after, seed=seed, )
def get_metric(period): return metrics.Rolling(metrics.MAE(), period)
(metrics.MicroRecall(), partial(sk_metrics.recall_score, average='micro')), (metrics.WeightedRecall(), partial(sk_metrics.recall_score, average='weighted')), (metrics.FBeta(beta=.5), partial(sk_metrics.fbeta_score, beta=.5)), (metrics.MacroFBeta(beta=.5), partial(sk_metrics.fbeta_score, beta=.5, average='macro')), (metrics.MicroFBeta(beta=.5), partial(sk_metrics.fbeta_score, beta=.5, average='micro')), (metrics.WeightedFBeta(beta=.5), partial(sk_metrics.fbeta_score, beta=.5, average='weighted')), (metrics.F1(), sk_metrics.f1_score), (metrics.MacroF1(), partial(sk_metrics.f1_score, average='macro')), (metrics.MicroF1(), partial(sk_metrics.f1_score, average='micro')), (metrics.WeightedF1(), partial(sk_metrics.f1_score, average='weighted')), (metrics.MCC(), sk_metrics.matthews_corrcoef), (metrics.MAE(), sk_metrics.mean_absolute_error), (metrics.MSE(), sk_metrics.mean_squared_error), ] @pytest.mark.parametrize('metric, sk_metric', [ pytest.param(metric, sk_metric, id=f'{metric.__class__.__name__}') for metric, sk_metric in TEST_CASES ]) @pytest.mark.filterwarnings('ignore::RuntimeWarning') @pytest.mark.filterwarnings( 'ignore::sklearn.metrics.classification.UndefinedMetricWarning') def test_metric(metric, sk_metric): # Check str works str(metric)
# construct a model pipeline model = compose.Select('clouds', 'humidity', 'pressure', 'temperature', 'wind') # To extract feature (bike number by hour) model += ( get_hour | feature_extraction.TargetAgg(by=['station', 'hour'], how=stats.Mean()) + feature_extraction.TargetAgg(by='station', how=stats.EWMean( 0.5)) # aggregate feature (station and time) ) model |= preprocessing.StandardScaler() model |= linear_model.LinearRegression() evaluate.progressive_val_score(dataset=datasets.Bikes(), model=model, metric=metrics.MAE(), moment='moment', delay=dt.timedelta(minutes=30), print_every=20_000) for x, y, in itertools.islice(X_y, 10000): y_pred = model.predict_one(x) model.learn_one(x, y) x, y = next(X_y) print(model.debug_one(x)) metrics = metrics.MAE()
def reset(self): self.mae = metrics.MAE() self.mse = metrics.MSE() self.r2 = metrics.R2() self.last_true_label = None self.last_prediction = None
# model = ( # preprocessing.StandardScaler() | # RadiusNeighborsRegressor(max_size=1000, r=1, aggregation='distance', # delta=0.1, seed=42, k=4) # ) model = (preprocessing.StandardScaler() | KNNRegressor(n_neighbors=3, window_size=1000, r=1, c=10, aggregation_method='mean', delta=0.1, seed=42, k=4)) print(f'L={model["KNNRegressor"]._buffer.L}') print(f'P1={model["KNNRegressor"]._buffer.P1}') print(f'P2={model["KNNRegressor"]._buffer.P2}') # exit() metric = metrics.MAE() evaluate.progressive_val_score(dataset, model, metric, print_every=100, show_time=True, show_memory=True)