def plot_performance(y_test, y_pred):
fig, ax1 = pyplot.subplots()
ax1.plot(y_test, y_test, 'g.', label=u'Optimum')
sigma = np.std(y_pred)
ax1.errorbar(y_test, y_pred.point(), yerr=sigma, label=u'Predictions', fmt='b.', ecolor='r', linewidth=0.5)
ax1.set_ylabel('Predicted $y_{pred}$')
ax1.set_xlabel('Correct label $y_{true}$')
ax1.legend(loc='best')
losses = log_loss(y_test, y_pred, sample=False)
ax2 = ax1.twinx()
overall = "{0:.2f}".format(np.mean(losses)) + " +/- {0:.2f}".format(np.std(losses))
ax2.set_ylabel('Loss')
ax2.plot(y_test, losses, 'y_', label=u'Loss: ' + overall)
ax2.tick_params(colors='y')
ax2.legend(loc=1)
pyplot.show()
def plot_performance(y_test, y_pred, filename=None):
"""
Visualises the prediction performance
Parameters
----------
y_test Ground truth
y_pred Predictions
filename If string, figure will be saved to file
Returns
-------
Matplotlib plot
"""
fig, ax1 = pyplot.subplots()
ax1.plot(y_test, y_test, 'g.', label=u'Optimum')
sigma = np.std(y_pred)
ax1.errorbar(y_test, y_pred.point(), yerr=sigma, label=u'Predictions', fmt='b.', ecolor='r', linewidth=0.5)
ax1.set_ylabel('Predicted $y_{pred}$')
ax1.set_xlabel('Correct label $y_{true}$')
ax1.legend(loc='best')
losses = log_loss(y_test, y_pred, sample=False)
ax2 = ax1.twinx()
overall = "{0:.2f}".format(np.mean(losses)) + " +/- {0:.2f}".format(np.std(losses))
ax2.set_ylabel('Loss')
ax2.plot(y_test, losses, 'y_', label=u'Loss: ' + overall)
ax2.tick_params(colors='y')
ax2.legend(loc=1)
if not isinstance(filename, str):
pyplot.show()
else:
pyplot.savefig(filename, transparent=True, bbox_inches='tight')
"""
with model:
# Priors
alpha = pm.Normal('alpha', mu=y.mean(), sd=10)
betas = pm.Normal('beta',
mu=0,
sd=10,
shape=X.get_value(borrow=True).shape[1])
sigma = pm.HalfNormal('sigma', sd=1)
# Model (defines y_pred)
mu = alpha + pm.math.dot(betas, X.T)
y_pred = pm.Normal("y_pred", mu=mu, sd=sigma, observed=y)
# Plug the model definition into the PyMC interface
model = BayesianVendorEstimator(model=PymcInterface(
model_definition=pymc_linear_regression))
# Run prediction, print and plot the results
data = DataManager('boston')
y_pred = model.fit(data.X_train, data.y_train).predict(data.X_test)
print('Log loss: ', log_loss(data.y_test, y_pred, return_std=True))
# Plot the performance
from ..utils import plot_performance
plot_performance(data.y_test, y_pred)
from sklearn.ensemble import RandomForestRegressor
from sklearn.datasets.base import load_boston
from sklearn.model_selection import train_test_split
from skpro.parametric import ParametricEstimator
from skpro.parametric.estimators import Constant
from skpro.metrics import log_loss
# Define the parametric model
model = ParametricEstimator(point=RandomForestRegressor(),
std=Constant('std(y)'),
shape='norm')
# Train and predict on boston housing data
X, y = load_boston(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
y_pred = model.fit(X_train, y_train).predict(X_test)
# Obtain the loss
loss = log_loss(y_test, y_pred, sample=True, return_std=True)
print('Loss: %f+-%f' % loss)
# Plot the performance
import sys
sys.path.append('../')
import utils
utils.plot_performance(y_test, y_pred)
""" Implements the point prediction """
return self.estimator.random_mean_prediction_
def std(self):
""" Implements the variance prediction """
return self.estimator.random_std_prediction_
def pdf(self, x):
""" Implements the pdf function """
return norm.pdf(x, loc=self.point()[self.index], scale=self.std()[self.index])
def __init__(self):
self.random_mean_prediction_ = None
self.random_std_prediction_ = None
def fit(self, X, y):
# Generate random parameter estimates
self.random_mean_prediction_ = randint(np.min(y), np.max(y))
self.random_std_prediction_ = 0.2 * self.random_mean_prediction_
return self
# Use custom model
model = MyCustomModel()
X, y = load_boston(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
y_pred = model.fit(X_train, y_train).predict(X_test)
print('Loss: %f+-%f' % log_loss(y_test, y_pred, return_std=True))
