def multiplexity_eval_metrics(sample: MultiLayerSplit,
pred_edges: pd.DataFrame) -> Dict[str, float]:
assert len(sample.layer_ids) == 2, 'binary case only yet'
layer_a, layer_b = sample.layer_ids
orig_mltplx = multiplexity(sample.full.edges,
layer_a,
layer_b,
reciprocal=False)
orig_mltrcp = multiplexity(sample.full.edges,
layer_a,
layer_b,
reciprocal=True)
pred_mltplx = multiplexity(pred_edges, layer_a, layer_b, reciprocal=False)
pred_mltrcp = multiplexity(pred_edges, layer_a, layer_b, reciprocal=True)
res = dict(mltplx_tgt=orig_mltplx,
mltplx_pred=pred_mltplx,
mltplx_mae=mean_absolute_error([orig_mltplx], [pred_mltplx]),
mltplx_mape=mean_absolute_percentage_error([orig_mltplx],
[pred_mltplx]),
mltrcp_mae=mean_absolute_error([orig_mltrcp], [pred_mltrcp]),
mltrcp_mape=mean_absolute_percentage_error([orig_mltrcp],
[pred_mltrcp]))
return res
def check_constraints(sample, probability_matrix):
s_in, s_out = empirical_strengths(sample.full.edges,
sample.full.nodes,
marginalized=True)
"""
Compute deviation of s_in/s_out from expected values.
Return dict with MAE and weighted MAPE.
"""
_s_in = probability_matrix.sum(axis=1)
_s_out = probability_matrix.sum(axis=0)
corr_in, pv_in = pearsonr(s_in, _s_in)
corr_out, pv_out = pearsonr(s_out, _s_out)
spcorr_in, sp_pv_in = spearmanr(s_in, _s_in)
spcorr_out, sp_pv_out = spearmanr(s_out, _s_out)
return dict(
s_in_mae=mean_absolute_error(s_in, _s_in),
s_out_mae=mean_absolute_error(s_out, _s_out),
s_in_mape=mean_absolute_percentage_error(s_in, _s_in),
s_out_mape=mean_absolute_percentage_error(s_out, _s_out),
# r2_in=r2_score(s_in, _s_in),
# r2_out=r2_score(s_out, _s_out),
s_in_js=jensenshannon(s_in, _s_in, base=2)**2,
s_out_js=jensenshannon(s_out, _s_out, base=2)**2,
corr_in=corr_in,
corr_out=corr_out,
spcorr_in=spcorr_in,
spcorr_out=spcorr_out,
pv_in=pv_in,
pv_out=pv_out,
sp_pv_in=sp_pv_in,
sp_pv_out=sp_pv_out,
)
def run_regression(train_embeds,
train_targets,
test_embeds,
test_targets,
scaler=None):
print('Ravelling train targets...')
train_targets_ravelled = train_targets.ravel()
print('Fitting squared loss Regressor...')
print(np.shape(train_embeds), np.shape(train_targets_ravelled))
regressor = SGDRegressor(loss="squared_loss", max_iter=1000, tol=1e-3)
regressor.fit(train_embeds, train_targets_ravelled)
print('Predicting outputs...')
tr_res = []
if scaler == None:
tr_res.append(
mean_squared_error(train_targets,
regressor.predict(train_embeds),
squared=False))
else:
rmse = mean_squared_error(train_targets,
regressor.predict(train_embeds),
squared=False)
tr_res.append(
scaler.inverse_transform(np.array(rmse).reshape(1, -1))[0][0])
tr_res.append(
mean_absolute_percentage_error(train_targets,
regressor.predict(train_embeds)))
tr_res.append(r2_score(train_targets, regressor.predict(train_embeds)))
tr_res.append(
explained_variance_score(train_targets,
regressor.predict(train_embeds)))
ts_res = []
if scaler == None:
ts_res.append(
mean_squared_error(test_targets,
regressor.predict(test_embeds),
squared=False))
else:
rmse = mean_squared_error(test_targets,
regressor.predict(test_embeds),
squared=False)
ts_res.append(
scaler.inverse_transform(np.array(rmse).reshape(1, -1))[0][0])
ts_res.append(
mean_absolute_percentage_error(test_targets,
regressor.predict(test_embeds)))
ts_res.append(r2_score(test_targets, regressor.predict(test_embeds)))
ts_res.append(
explained_variance_score(test_targets, regressor.predict(test_embeds)))
return tr_res, ts_res
def test_deprecation_positional_arguments_mape():
y_true = [1, 1, 1]
y_pred = [1, 0, 1]
sample_weights = [0.5, 0.1, 0.2]
multioutput = "raw_values"
warning_msg = "passing these as positional arguments will result in an error"
# Trigger the warning
with pytest.warns(FutureWarning, match=warning_msg):
mean_absolute_percentage_error(y_true, y_pred, sample_weights,
multioutput)
def binary_classification_metrics(target: Iterable[Edge],
pred: Iterable[Edge]):
target = set(target)
pred = set(pred)
num_expected = len(target)
num_predicted = len(pred)
L_mape = mean_absolute_percentage_error([num_expected], [num_predicted])
tp = len(target & pred)
fp = len(pred - target)
fn = len(target - pred)
# tn = n * n - (tp + fp + fn)
if num_predicted != 0:
precision = tp / (tp + fp)
else:
precision = 1
if num_expected != 0:
recall = tp / (tp + fn)
else:
recall = 1
if precision + recall > 0:
f1 = 2 * precision * recall / (precision + recall)
else:
f1 = 0
return dict(precision=precision,
recall=recall,
f1=f1,
num_expected=num_expected,
num_predicted=num_predicted,
L_mape=L_mape)
def objective(trial):
param = params.copy()
param.update({
'eta':
trial.suggest_loguniform('eta', 0.01, 1),
'lambda':
trial.suggest_loguniform('lambda', 1e-4, 1),
'alpha':
trial.suggest_loguniform('alpha', 1e-4, 1.0),
'gamma':
trial.suggest_categorical('gamma', [0, 1, 5, 20, 70]),
'max_depth':
trial.suggest_int('max_depth', 2, 5),
'min_child_weight':
trial.suggest_categorical('min_child_weight', [0.5, 1, 5, 8, 20, 50]),
'subsample':
trial.suggest_loguniform('subsample', 1e-4, 1),
'colsample_bytree':
trial.suggest_loguniform('colsample_bytree', 1e-4, 1),
})
model = xgb.train(param,
data,
num_boost_round=50000,
verbose_eval=False,
evals=[(X_test, "Test")],
early_stopping_rounds=500)
err = mean_absolute_percentage_error(X_test.get_label(),
model.predict(X_test))
return err
def metrics_stat(y_true: np.array, y_pred: np.array) -> typing.Dict[str,float]:
mape = mean_absolute_percentage_error(y_true, y_pred)
mdape = median_absolute_percentage_error(y_true, y_pred)
rmse = mean_squared_error(y_true, y_pred, squared=False)
r2 = r2_score(y_true, y_pred)
raif_metric = deviation_metric(y_true, y_pred)
return {'mape':mape, 'mdape':mdape, 'rmse': rmse, 'r2': r2, 'raif_metric':raif_metric}
def update_model(self, X: np.ndarray, y: np.ndarray):
# NOTE: the GPR model will be created since the effective search space (the reduced space
# is dynamic)
dim = self._search_space.dim
bounds = np.asarray(self._search_space.bounds)
self.model = GaussianProcess(
mean=trend.constant_trend(dim),
corr="matern",
thetaL=1e-3 * (bounds[:, 1] - bounds[:, 0]),
thetaU=1e3 * (bounds[:, 1] - bounds[:, 0]),
nugget=1e-6,
noise_estim=False,
optimizer="BFGS",
wait_iter=3,
random_start=max(10, dim),
likelihood="concentrated",
eval_budget=100 * dim,
)
_std = np.std(y)
y_ = y if np.isclose(_std, 0) else (y - np.mean(y)) / _std
self.fmin, self.fmax = np.min(y_), np.max(y_)
self.frange = self.fmax - self.fmin
self.model.fit(X, y_)
y_hat = self.model.predict(X)
r2 = r2_score(y_, y_hat)
MAPE = mean_absolute_percentage_error(y_, y_hat)
self.logger.info(f"model r2: {r2}, MAPE: {MAPE}")
def create_accuracy_plot_and_return_mape_and_cumulative_prod(result_df):
prediction_array = result_df[
'2-Year Cumulative Production (bbl)'].to_numpy() / 1000
solution_array = result_df['Total_Prod_Well'].to_numpy() / 1000
plt.figure(figsize=(6, 4))
plt.plot(solution_array, prediction_array, 'o', label='Estimates')
plt.plot([0, 1500], [0, 1500], '--r', label='1:1 line')
plt.plot([solution_array[0], solution_array[0]],
[prediction_array[0], solution_array[0]],
'--',
color='gray',
label='misfit')
for i in range(1, 3):
plt.plot([solution_array[i], solution_array[i]],
[prediction_array[i], solution_array[i]],
'--',
color='gray')
plt.xlabel('True, MSTB')
plt.ylabel('Prediction, MSTB')
plt.grid('on')
plt.legend()
plt.axis([0, 1500, 0, 1500])
plt.savefig('accuracy.pgf')
mape = np.round(
mean_absolute_percentage_error(solution_array, prediction_array), 5)
total_prod = result_df['Total_Prod_Field'].to_numpy()[0]
return mape, total_prod
def update_model(self):
# TODO: implement a proper model selection here
# TODO: in case r2 is really poor, re-fit the model or log-transform `fitness`?
data = self.data
fitness = data.fitness
# TODO: to standardize the response values to prevent numerical overflow that might
# appear in the MGF-based acquisition function.
# Standardization should make it easier to specify the GP prior, compared to
# rescaling values to the unit interval.
_std = np.std(fitness)
if len(fitness) > 5 and np.isclose(_std, 0):
raise FlatFitnessError()
fitness_ = fitness if np.isclose(
_std, 0) else (fitness - np.mean(fitness)) / np.std(fitness)
self.fmin, self.fmax = np.min(fitness_), np.max(fitness_)
self.frange = self.fmax - self.fmin
self.model.fit(data, fitness_.reshape(-1, 1))
fitness_hat = self.model.predict(data)
r2 = r2_score(fitness_, fitness_hat)
MAPE = mean_absolute_percentage_error(fitness_, fitness_hat)
self.logger.info(f"model r2: {r2}, MAPE: {MAPE}")
def run(self, duration, name, interval, plot=False, record=True, verbose=True):
begin = time.time()
self._warmStart()
self.cur = self.startPont
acceptable = False
# streaming
while time.time() - begin < duration and self.nxt < len(self.yTrue):
time.sleep(interval)
self._update()
self._predict()
# evaluate model
if self.lazy and self.cur > 12:
acceptable = self._evaluate('mape', 12, 0.1)
if not acceptable:
self._learnMany(numberOfData=12)
else:
self._learnOne()
if plot:
plotlyplot(actual=self.actualList, prediction=self.predList,
times=self.times[:self.cur-1], plotname=name)
if record:
dict = {'actual': self.actualList, 'predict': self.predList, 'time': self.timeline}
df = pd.DataFrame(dict)
df.to_csv('/Users/cicada/Documents/DTU_resource/Thesis/Incremental-learning-EL/src/results-'+name + '.csv')
# print(df.head())
if verbose:
print(mean_absolute_percentage_error(self.actualList[200:], self.predList[200:]))
def eval_model(model, pred_step, counts):
model = model[::pred_step]
counts = counts[::pred_step]
lbs = [5, 5, 5, 5, 5, 5, 5, 5, 5]
ubs = [5, 5, 5, 5, 5, 5, 5, 5, 5]
conf = 0.95
print("Total Values: {}".format(min(len(model), len(counts))))
print("Forecast: {}".format(model))
print("Validation Set: {}".format(counts))
print()
mae = mean_absolute_error(model, counts)
rmse = math.sqrt(mean_squared_error(model, counts))
mape = mean_absolute_percentage_error(model, counts)
smape = symmetric_mean_absolute_percentage_error(model, counts)
mase = mean_absolute_scaled_error(model, counts)
# ias = interval_accuracy_score(counts, lbs, ubs, conf)
print("Mean Absolute Error: {}".format(mae))
print("Root Mean Squared Error: {}".format(rmse))
print("Mean Absolute Percentage Error: {}".format(mape))
print("Symmetric Mean Absolute Percentage Error: {}".format(smape))
print("Mean Absolute Scaled Error: {}".format(mase))
# print("Mean Interval Accuracy Score: {}".format(ias))
return rmse
def main():
estimators = [('lr', Ridge())]
backup = [('gb', GradientBoostingRegressor()),
('rf', RandomForestRegressor())]
run = TrainPredictDuration('resource/config.ini', 'model')
subscr_type = run.subscr_type
df = run.data_preparation()
df = run.feature_engineering(df)
X, y, feature_names = run.feature_selection(df)
print(feature_names)
for estimator in estimators:
model_name = estimator[0] + '_{}'.format(subscr_type)
model, y_pred, y_test = run.grid_search_train(X, y, estimator[1],
model_name)
print('{}-MSE: '.format(model_name),
mean_squared_error(y_test, y_pred))
print('{}-MAPE: '.format(model_name),
mean_absolute_percentage_error(y_test, y_pred))
print('{}-R2: '.format(model_name), r2_score(y_test, y_pred))
run.save_model(model, '{}.sav'.format(model_name))
run.save_output(y_pred, y_test, '{}.json'.format(model_name))
if model_name == 'lr' + '_{}'.format(subscr_type):
for idx, value in enumerate(feature_names):
print(value + ': ', round(model.coef_[idx], 5))
def plotHoltWinters(series, plot_intervals=False, plot_anomalies=False):
"""
series - dataset with timeseries
plot_intervals - show confidence intervals
plot_anomalies - show anomalies
"""
plt.figure(figsize=(20, 10))
plt.plot(model.result, label = "Model")
plt.plot(series.values, label = "Actual")
error = mean_absolute_percentage_error(series.values, model.result[:len(series)])
plt.title("Mean Absolute Percentage Error: {0:.2f}%".format(error))
if plot_anomalies:
anomalies = np.array([np.NaN]*len(series))
anomalies[series.values<model.LowerBond[:len(series)]] = \
series.values[series.values<model.LowerBond[:len(series)]]
anomalies[series.values>model.UpperBond[:len(series)]] = \
series.values[series.values>model.UpperBond[:len(series)]]
plt.plot(anomalies, "o", markersize=10, label = "Anomalies")
if plot_intervals:
plt.plot(model.UpperBond, "r--", alpha=0.5, label = "Up/Low confidence")
plt.plot(model.LowerBond, "r--", alpha=0.5)
plt.fill_between(x=range(0,len(model.result)), y1=model.UpperBond,
y2=model.LowerBond, alpha=0.2, color = "grey")
plt.vlines(len(series), ymin=min(model.LowerBond), ymax=max(model.UpperBond), linestyles='dashed')
plt.axvspan(len(series)-20, len(model.result), alpha=0.3, color='lightgrey')
plt.grid(True)
plt.axis('tight')
plt.legend(loc="best", fontsize=13);
def evaluate(self):
r2 = r2_score(self.test_y, self.y_pred)
rmse = mean_squared_error(self.test_y, self.y_pred, squared=False)
mae = mean_absolute_error(self.test_y, self.y_pred)
mape = mean_absolute_percentage_error(self.test_y, self.y_pred)
return r2, rmse, mae, mape
def calc_metrics(y_true, y_pred):
"""
This Function calculates all relevant metrics for model evaluation.
:param y_true: Array-like list of true values.
:param y_pred: Array-like list of predicted values.
"""
logger.info("Start calc_metrics()")
# calc metrics
mae = met.mean_absolute_error(y_true, y_pred)
mse = met.mean_squared_error(y_true, y_pred)
rmse = met.mean_squared_error(y_true, y_pred, squared=False)
mape = met.mean_absolute_percentage_error(y_true, y_pred)
r2 = met.r2_score(y_true, y_pred)
# combine results into dataframe
metrics_series = [
mae,
mse,
rmse # , mape
,
r2
]
# clean small values, so SQL can parse str to numeric
metrics_series = [round(num, 4) for num in metrics_series]
return metrics_series
def eval_model(predictions, pred_step, actual, verbose=False):
predictions = predictions[::pred_step]
actual = actual[::pred_step]
mae = mean_absolute_error(actual, predictions)
rmse = math.sqrt(mean_squared_error(actual, predictions))
ignore_zero_values = np.where(np.array(actual) == 0, 0, 1)
mape = mean_absolute_percentage_error(actual, predictions,
ignore_zero_values)
smape = symmetric_mean_absolute_percentage_error(actual, predictions)
mase = mean_absolute_scaled_error(actual, predictions)
ias = interval_accuracy_score(actual, predictions)
if verbose:
print("Total Values: {}".format(min(len(predictions), len(actual))))
print("Forecast: {}".format(predictions))
print("Validation Set: {}".format(actual))
print()
print("Mean Absolute Error: {}".format(mae))
print("Root Mean Squared Error: {}".format(rmse))
print("Mean Absolute Percentage Error: {}".format(mape))
print("Symmetric Mean Absolute Percentage Error: {}".format(smape))
print("Mean Absolute Scaled Error: {}".format(mase))
print("Mean Interval Accuracy Score: {}".format(ias))
metrics = dict(mae=mae,
rmse=rmse,
mape=mape,
smape=smape,
mase=mase,
ias=ias)
return metrics
def get_metrics(y_true, y_pred):
""" Calculates all desired metrics for a given set of predictions and ground truths.
Parameters:
-----------
y_true : ndarray, shape [N]
Ground truth class labels.
y_pred : ndarray, shape [N]
Predicted class probabilities or hard labels.
Returns:
--------
metrics : defaultdict
A dict containing values for all metrics.
"""
metrics = defaultdict(float)
#metrics['accuracy'] = accuracy_score(y_true, y_pred >= .5)
metrics['accuracy'] = mean_absolute_percentage_error(y_true, y_pred)
try:
metrics['auc'] = roc_auc_score(y_true, y_pred)
except:
# AUC is not defined, if only one class is present
metrics['auc'] = np.nan
metrics['ppr'] = (y_pred >= .5).sum() / y_pred.shape[0]
return metrics
def evaluate_forecast(self):
n = min(len(self.validation_data), len(self.forecasts))
y_forecast = self.forecasts[:n]
y_actual = self.validation_data.tail(n)["close"]
mean_abs_err = learn.mean_absolute_error(y_actual, y_forecast)
mean_sq_err = learn.mean_squared_error(y_actual, y_forecast)
mean_sq_lg_err = learn.mean_squared_log_error(y_actual, y_forecast)
mean_abs_percent_err = learn.mean_absolute_percentage_error(
y_actual, y_forecast)
median_abs_err = learn.median_absolute_error(y_actual, y_forecast)
mean_gamma_dev = learn.mean_gamma_deviance(y_actual, y_forecast)
mean_poisson_dev = learn.mean_poisson_deviance(y_actual, y_forecast)
mean_tweedie_dev = learn.mean_tweedie_deviance(y_actual, y_forecast)
explained_variance = learn.explained_variance_score(
y_actual, y_forecast)
max_residual = learn.max_error(y_actual, y_forecast)
coeff_determination = learn.r2_score(y_actual, y_forecast)
metrics = {
"Mean Squared Error (MSE)": mean_sq_err,
"Mean Absolute Error (MAE)": mean_abs_err,
"Mean Squared Logarithmic Error (MSLE)": mean_sq_lg_err,
"Mean Absolute Percentage Error (MAPE)": mean_abs_percent_err,
"Median Absolute Error (MedAE)": median_abs_err,
"Mean Gamma Deviance": mean_gamma_dev,
"Mean Poisson Deviance": mean_poisson_dev,
"Mean Tweedie Deviance Error": mean_tweedie_dev,
"Explained Variance Regression Score": explained_variance,
"Max Residual Error": max_residual,
"Coefficient of Determination": coeff_determination
}
self.metrics = metrics
def calc_performance_metrics(
labels: np.ndarray,
predictions: np.ndarray,
decimal_points: Optional[int] = 4) -> _PerformanceMetrics:
"""Calculates performance metrics related to a regression model.
Args:
labels: An array of true labels containing numeric values.
predictions: An array of predictions containing numeric values.
decimal_points: Number of decimal points to use when outputting the
calculated performance metrics.
Returns:
Object of _PerformanceMetrics class containing the regression diagnostics
metrics.
"""
utils.assert_label_and_prediction_length_match(labels, predictions)
mse = metrics.mean_squared_error(labels, predictions)
rmse = np.sqrt(mse)
mae = metrics.mean_absolute_error(labels, predictions)
mape = metrics.mean_absolute_percentage_error(labels, predictions)
r2 = metrics.r2_score(labels, predictions)
corr = sp.stats.pearsonr(labels, predictions)[0]
return _PerformanceMetrics(
mean_squared_error=round(mse, decimal_points),
root_mean_squared_error=round(rmse, decimal_points),
mean_absolute_error=round(mae, decimal_points),
mean_absolute_percentage_error=round(mape, decimal_points),
r_squared=round(r2, decimal_points),
pearson_correlation=round(corr, decimal_points))
def print_score():
df = get_pure_cases_df()
X, y = sliding_window(df, 28, 14)
assert X.shape[0] == y.shape[0]
X = X.numpy()
y = y.numpy()
mae = []
rmse = []
mape = []
for i in range(X.shape[0]):
mae.append(
mean_absolute_error(y[i], np.broadcast_to(X[i][-1],
(y.shape[1], 1))))
rmse.append(
mean_squared_error(y[i],
np.broadcast_to(X[i][-1], (y.shape[1], 1)),
squared=False))
mape.append(
mean_absolute_percentage_error(
y[i], np.broadcast_to(X[i][-1], (y.shape[1], 1))))
mae = np.array(mae)
rmse = np.array(rmse)
mape = np.array(mape)
print(f"RMSE: {rmse.mean()}, MAE: {mae.mean()}, MAPE: {mape.mean()}")
def test_regression_custom_weights():
y_true = [[1, 2], [2.5, -1], [4.5, 3], [5, 7]]
y_pred = [[1, 1], [2, -1], [5, 4], [5, 6.5]]
msew = mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6])
rmsew = mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6], squared=False)
maew = mean_absolute_error(y_true, y_pred, multioutput=[0.4, 0.6])
mapew = mean_absolute_percentage_error(y_true, y_pred, multioutput=[0.4, 0.6])
rw = r2_score(y_true, y_pred, multioutput=[0.4, 0.6])
evsw = explained_variance_score(y_true, y_pred, multioutput=[0.4, 0.6])
evsw2 = explained_variance_score(
y_true, y_pred, multioutput=[0.4, 0.6], force_finite=False
)
assert_almost_equal(msew, 0.39, decimal=2)
assert_almost_equal(rmsew, 0.59, decimal=2)
assert_almost_equal(maew, 0.475, decimal=3)
assert_almost_equal(mapew, 0.1668, decimal=2)
assert_almost_equal(rw, 0.94, decimal=2)
assert_almost_equal(evsw, 0.94, decimal=2)
assert_almost_equal(evsw2, 0.94, decimal=2)
# Handling msle separately as it does not accept negative inputs.
y_true = np.array([[0.5, 1], [1, 2], [7, 6]])
y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]])
msle = mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7])
msle2 = mean_squared_error(
np.log(1 + y_true), np.log(1 + y_pred), multioutput=[0.3, 0.7]
)
assert_almost_equal(msle, msle2, decimal=2)
def test_multioutput_regression():
y_true = np.array([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
y_pred = np.array([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
error = mean_squared_error(y_true, y_pred)
assert_almost_equal(error, (1. / 3 + 2. / 3 + 2. / 3) / 4.)
error = mean_squared_error(y_true, y_pred, squared=False)
assert_almost_equal(error, 0.454, decimal=2)
error = mean_squared_log_error(y_true, y_pred)
assert_almost_equal(error, 0.200, decimal=2)
# mean_absolute_error and mean_squared_error are equal because
# it is a binary problem.
error = mean_absolute_error(y_true, y_pred)
assert_almost_equal(error, (1. + 2. / 3) / 4.)
error = np.around(mean_absolute_percentage_error(y_true, y_pred),
decimals=2)
assert np.isfinite(error)
assert error > 1e6
error = median_absolute_error(y_true, y_pred)
assert_almost_equal(error, (1. + 1.) / 4.)
error = r2_score(y_true, y_pred, multioutput='variance_weighted')
assert_almost_equal(error, 1. - 5. / 2)
error = r2_score(y_true, y_pred, multioutput='uniform_average')
assert_almost_equal(error, -.875)
def test_regression_metrics(n_samples=50):
y_true = np.arange(n_samples)
y_pred = y_true + 1
assert_almost_equal(mean_squared_error(y_true, y_pred), 1.)
assert_almost_equal(
mean_squared_log_error(y_true, y_pred),
mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred)))
assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.)
assert_almost_equal(median_absolute_error(y_true, y_pred), 1.)
mape = mean_absolute_percentage_error(y_true, y_pred)
assert np.isfinite(mape)
assert mape > 1e6
assert_almost_equal(max_error(y_true, y_pred), 1.)
assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), 1.)
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=0),
mean_squared_error(y_true, y_pred))
# Tweedie deviance needs positive y_pred, except for p=0,
# p>=2 needs positive y_true
# results evaluated by sympy
y_true = np.arange(1, 1 + n_samples)
y_pred = 2 * y_true
n = n_samples
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=-1),
5 / 12 * n * (n**2 + 2 * n + 1))
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=1),
(n + 1) * (1 - np.log(2)))
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=2),
2 * np.log(2) - 1)
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3 / 2),
((6 * np.sqrt(2) - 8) / n) * np.sqrt(y_true).sum())
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3),
np.sum(1 / y_true) / (4 * n))
def test_regression_metrics_at_limits():
assert_almost_equal(mean_squared_error([0.], [0.]), 0.00, 2)
assert_almost_equal(mean_squared_error([0.], [0.], squared=False), 0.00, 2)
assert_almost_equal(mean_squared_log_error([0.], [0.]), 0.00, 2)
assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2)
assert_almost_equal(mean_absolute_percentage_error([0.], [0.]), 0.00, 2)
assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2)
assert_almost_equal(max_error([0.], [0.]), 0.00, 2)
assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2)
assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2)
err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=err_msg):
mean_squared_log_error([-1.], [-1.])
err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=err_msg):
mean_squared_log_error([1., 2., 3.], [1., -2., 3.])
err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=err_msg):
mean_squared_log_error([1., -2., 3.], [1., 2., 3.])
# Tweedie deviance error
power = -1.2
assert_allclose(mean_tweedie_deviance([0], [1.], power=power),
2 / (2 - power), rtol=1e-3)
with pytest.raises(ValueError,
match="can only be used on strictly positive y_pred."):
mean_tweedie_deviance([0.], [0.], power=power)
assert_almost_equal(mean_tweedie_deviance([0.], [0.], power=0), 0.00, 2)
msg = "only be used on non-negative y and strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.], [0.], power=1.0)
power = 1.5
assert_allclose(mean_tweedie_deviance([0.], [1.], power=power),
2 / (2 - power))
msg = "only be used on non-negative y and strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.], [0.], power=power)
power = 2.
assert_allclose(mean_tweedie_deviance([1.], [1.], power=power), 0.00,
atol=1e-8)
msg = "can only be used on strictly positive y and y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.], [0.], power=power)
power = 3.
assert_allclose(mean_tweedie_deviance([1.], [1.], power=power),
0.00, atol=1e-8)
msg = "can only be used on strictly positive y and y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.], [0.], power=power)
with pytest.raises(ValueError,
match="is only defined for power<=0 and power>=1"):
mean_tweedie_deviance([0.], [0.], power=0.5)
def get_glm_prediction():
confirmed_cases_url = "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv" # noqa
confirmed_cases = pd.read_csv(confirmed_cases_url, sep=",")
gg = confirmed_cases[confirmed_cases["Country/Region"] == "India"]
gg = gg.T
gg["cases"] = gg[147]
df = gg.drop(147, axis=1).iloc[4:]
df = df.diff().iloc[1:]
rmse = []
mae = []
mapes = []
arr = df["cases"].to_numpy(dtype=np.float64)
df["one_old"] = df["cases"].shift(1)
df["two_old"] = df["cases"].shift(2)
df["three_old"] = df["cases"].shift(3)
df = df[5:]
x_train = df[["one_old", "two_old", "three_old"]]
y_train = df["cases"]
x_test = x_train[275:300]
y_test = y_train[275:300]
x_train = x_train[:275]
y_train = y_train[:275]
xtrain = x_train.to_numpy(dtype=np.float64)
ytrain = y_train.to_numpy(dtype=np.float64)
xtest = x_test.to_numpy(dtype=np.float64)
ytest = y_test.to_numpy(dtype=np.float64)
poisson_training_results = sm.GLM(ytrain[:], xtrain[:, :]).fit()
poisson_predictions = poisson_training_results.get_prediction(xtest)
predictions_summary_frame = poisson_predictions.summary_frame()
predictions_summary_frame["Actual"] = ytest
predictions_summary_frame["mean"] = predictions_summary_frame["mean"]
for i in range(14):
xtest = np.hstack(
(np.array(predictions_summary_frame["mean"]).reshape(-1, 1),
xtest[:, :-1]))
ytest = np.vstack((
ytest[1:].reshape(-1, 1),
(np.array(df["cases"][300 + i:300 + i + 1],
dtype="float")).reshape(-1, 1),
))
predictions_summary_frame[
"mean"] = poisson_training_results.get_prediction(
xtest).summary_frame()["mean"]
rmse.append(
mean_squared_error((predictions_summary_frame["mean"]), (ytest),
squared=False))
mae.append(
mean_absolute_error((predictions_summary_frame["mean"]), (ytest)))
mapes.append(
mean_absolute_percentage_error(predictions_summary_frame["mean"],
ytest))
rmse = np.array(rmse)
mae = np.array(mae)
mapes = np.array(mapes)
print(f"RMSE: {rmse.mean()}, MAE: {mae.mean()}, MAPE: {mapes.mean()}")
def test_regression_metrics(n_samples=50):
y_true = np.arange(n_samples)
y_pred = y_true + 1
y_pred_2 = y_true - 1
assert_almost_equal(mean_squared_error(y_true, y_pred), 1.0)
assert_almost_equal(
mean_squared_log_error(y_true, y_pred),
mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred)),
)
assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.0)
assert_almost_equal(mean_pinball_loss(y_true, y_pred), 0.5)
assert_almost_equal(mean_pinball_loss(y_true, y_pred_2), 0.5)
assert_almost_equal(mean_pinball_loss(y_true, y_pred, alpha=0.4), 0.6)
assert_almost_equal(mean_pinball_loss(y_true, y_pred_2, alpha=0.4), 0.4)
assert_almost_equal(median_absolute_error(y_true, y_pred), 1.0)
mape = mean_absolute_percentage_error(y_true, y_pred)
assert np.isfinite(mape)
assert mape > 1e6
assert_almost_equal(max_error(y_true, y_pred), 1.0)
assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), 1.0)
assert_almost_equal(
mean_tweedie_deviance(y_true, y_pred, power=0),
mean_squared_error(y_true, y_pred),
)
assert_almost_equal(d2_tweedie_score(y_true, y_pred, power=0),
r2_score(y_true, y_pred))
# Tweedie deviance needs positive y_pred, except for p=0,
# p>=2 needs positive y_true
# results evaluated by sympy
y_true = np.arange(1, 1 + n_samples)
y_pred = 2 * y_true
n = n_samples
assert_almost_equal(
mean_tweedie_deviance(y_true, y_pred, power=-1),
5 / 12 * n * (n**2 + 2 * n + 1),
)
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=1),
(n + 1) * (1 - np.log(2)))
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=2),
2 * np.log(2) - 1)
assert_almost_equal(
mean_tweedie_deviance(y_true, y_pred, power=3 / 2),
((6 * np.sqrt(2) - 8) / n) * np.sqrt(y_true).sum(),
)
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3),
np.sum(1 / y_true) / (4 * n))
dev_mean = 2 * np.mean(xlogy(y_true, 2 * y_true / (n + 1)))
assert_almost_equal(
d2_tweedie_score(y_true, y_pred, power=1),
1 - (n + 1) * (1 - np.log(2)) / dev_mean,
)
dev_mean = 2 * np.log((n + 1) / 2) - 2 / n * np.log(factorial(n))
assert_almost_equal(d2_tweedie_score(y_true, y_pred, power=2),
1 - (2 * np.log(2) - 1) / dev_mean)
def metric_by_category(df):
r2 = r2_score(df[constants.label_header], df['pred'])
mape = mean_absolute_percentage_error(df[constants.label_header],
df['pred'])
mae = mean_absolute_error(df[constants.label_header], df['pred'])
rmse = np.sqrt(
mean_squared_error(df[constants.label_header], df['pred']))
return pd.Series(dict(r2=r2, mape=mape, mae=mae, rmse=rmse))
def test_regression_multioutput_array():
y_true = [[1, 2], [2.5, -1], [4.5, 3], [5, 7]]
y_pred = [[1, 1], [2, -1], [5, 4], [5, 6.5]]
mse = mean_squared_error(y_true, y_pred, multioutput='raw_values')
mae = mean_absolute_error(y_true, y_pred, multioutput='raw_values')
mape = mean_absolute_percentage_error(y_true,
y_pred,
multioutput='raw_values')
r = r2_score(y_true, y_pred, multioutput='raw_values')
evs = explained_variance_score(y_true, y_pred, multioutput='raw_values')
assert_array_almost_equal(mse, [0.125, 0.5625], decimal=2)
assert_array_almost_equal(mae, [0.25, 0.625], decimal=2)
assert_array_almost_equal(mape, [0.0778, 0.2262], decimal=2)
assert_array_almost_equal(r, [0.95, 0.93], decimal=2)
assert_array_almost_equal(evs, [0.95, 0.93], decimal=2)
# mean_absolute_error and mean_squared_error are equal because
# it is a binary problem.
y_true = [[0, 0]] * 4
y_pred = [[1, 1]] * 4
mse = mean_squared_error(y_true, y_pred, multioutput='raw_values')
mae = mean_absolute_error(y_true, y_pred, multioutput='raw_values')
r = r2_score(y_true, y_pred, multioutput='raw_values')
assert_array_almost_equal(mse, [1., 1.], decimal=2)
assert_array_almost_equal(mae, [1., 1.], decimal=2)
assert_array_almost_equal(r, [0., 0.], decimal=2)
r = r2_score([[0, -1], [0, 1]], [[2, 2], [1, 1]], multioutput='raw_values')
assert_array_almost_equal(r, [0, -3.5], decimal=2)
assert np.mean(r) == r2_score([[0, -1], [0, 1]], [[2, 2], [1, 1]],
multioutput='uniform_average')
evs = explained_variance_score([[0, -1], [0, 1]], [[2, 2], [1, 1]],
multioutput='raw_values')
assert_array_almost_equal(evs, [0, -1.25], decimal=2)
# Checking for the condition in which both numerator and denominator is
# zero.
y_true = [[1, 3], [-1, 2]]
y_pred = [[1, 4], [-1, 1]]
r2 = r2_score(y_true, y_pred, multioutput='raw_values')
assert_array_almost_equal(r2, [1., -3.], decimal=2)
assert np.mean(r2) == r2_score(y_true,
y_pred,
multioutput='uniform_average')
evs = explained_variance_score(y_true, y_pred, multioutput='raw_values')
assert_array_almost_equal(evs, [1., -3.], decimal=2)
assert np.mean(evs) == explained_variance_score(y_true, y_pred)
# Handling msle separately as it does not accept negative inputs.
y_true = np.array([[0.5, 1], [1, 2], [7, 6]])
y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]])
msle = mean_squared_log_error(y_true, y_pred, multioutput='raw_values')
msle2 = mean_squared_error(np.log(1 + y_true),
np.log(1 + y_pred),
multioutput='raw_values')
assert_array_almost_equal(msle, msle2, decimal=2)
def _mape(self):
"""
Function to calculate mean-absolute-percentage-error
"""
mape = mean_absolute_percentage_error(y_true=self.y_true,
y_pred=self.y_pred,
multioutput=self.multioutput)
return mape
