def compare_models(stats, path_spec, ylimr=(-0.3, 0.25), title="Final Model Comparison"):
df = pd.DataFrame.from_dict(stats, orient='index')
df.columns = ['mean absolute error', '$R^2$']
sns.set_style(style='whitegrid')
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5), gridspec_kw={'top': 0.92})
plt.suptitle(title)
bars = sns.barplot(x=df.index, y=df['mean absolute error'], ax=ax1, alpha=0.8)
for p in bars.patches:
bars.annotate(format(int(p.get_height()), ',d'), (p.get_x() + p.get_width() / 2., p.get_height()), ha='center',
va='center', xytext=(0, 10), textcoords='offset points')
ax1.set_ylim((3.5e7, 5e7))
bars = sns.barplot(x=df.index, y=df['$R^2$'], ax=ax2, alpha=0.8)
for p in bars.patches:
if p.get_height() > 0:
bars.annotate(format(p.get_height(), '.4f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha='center',
va='center', xytext=(0, 10), textcoords='offset points')
else:
bars.annotate(format(p.get_height(), '.4f'), (p.get_x() + p.get_width() / 2., p.get_height() - 0.04),
ha='center',
va='center', xytext=(0, 10), textcoords='offset points')
ax2.set_ylim(ylimr)
handle_savefig(path_spec)
def plot_predictions(y_true, y_pred, n=10, savefig=None):
idx = y_true.ravel().argsort(axis=0)
y_true = y_true[idx]
y_pred = y_pred[idx]
fig = plt.figure(figsize=(10, 10))
plt.scatter(x=y_true[:n], y=y_pred[:n], marker='x', color='black')
plt.xlabel('True viewcounts')
plt.ylabel('Predicted viewcounts')
plt.yscale(plt.gca().get_xscale())
plt.yticks(plt.gca().get_xticks())
plt.ylim(plt.gca().get_xlim())
handle_savefig(savefig)
def plot_sorted_residuals(path,
data_dict=FULL_DATA_DICT,
model_info=None,
savefig=None,
log_tf=True):
pipeline = ut.load_model(path)
X_train, X_test, y_train, y_test = DataManager.load_tts_data(**data_dict)
y_pred = pipeline.predict(X_test)
if check_log_tf(path):
y_pred = DataManager.inv_tf(y_pred)
y_test = np.array(y_test).ravel()
sorted_idx = y_test.argsort()
y_test_srt = y_test[sorted_idx]
y_pred_srt = y_pred[sorted_idx]
if log_tf:
y_test_srt, y_pred_srt = DataManager.log_tf(y_test_srt, y_pred_srt)
fig, (ax1, ax2, ax3) = plt.subplots(3,
1,
figsize=(15, 10),
gridspec_kw={'top': 0.90})
x = range(len(y_test_srt))
mape_tf = mean_absolute_percentage_error(y_test_srt, y_pred_srt)
mape = mean_absolute_percentage_error(y_test, y_pred)
print(mape)
plt.suptitle('Sorted Viewcount error comparison:\n' +
'MAPE Transformed: {:.2%}\n'.format(mape_tf / 100) +
'MAPE: {:.2%}\n'.format(mape / 100) +
generate_model_descr(pipeline, model_info))
plt.tight_layout()
ax1.scatter(x=x, y=y_test_srt, s=1.5, alpha=0.9, marker='x')
ax2.scatter(x=x, y=y_pred_srt, s=1.5, alpha=0.9, marker='x')
ax2.set_ylabel('predicted log(viewcounts)' if log_tf else 'viewcounts')
ax1.set_ylabel('true log(viewcounts)' if log_tf else 'viewcounts')
ax2.set_ylim(ax1.get_ylim())
ax2.set_xticks([])
ax1.set_xticks([])
rel_error = np.abs((y_test_srt - y_pred_srt) / y_test_srt)
ax3.scatter(x=x, y=rel_error, s=1.5, alpha=0.9, marker='x', c='red')
# ax3.set_ylim((0.0, 100.00))
ax3.set_ylabel('Relative Error')
ax3.set_xlabel('Sorted Sample Index')
ut.handle_savefig(savefig)
return mean_absolute_percentage_error(y_test, y_pred)
def plot_prediction_results(y_test, y_pred, loss_function=mean_absolute_error, savefig=None, title='model'):
err = loss_function(y_test, y_pred)
fig, ax = plt.subplots(2, 4, figsize=(20, 10), gridspec_kw={'hspace': 0.2})
plt.suptitle(f'Test/Pred of {title}: {loss_function.__name__}={err}')
ax[0, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
ax[0, 0].set_title('Test set distribution')
ax[0, 1].hist(y_pred.ravel(), bins=25)
ax[0, 0].set_yscale('log')
ax[0, 1].set_yscale('log')
ax[0, 0].set_ylabel('count (log)')
ax[0, 1].set_ylabel('count (log)')
ax[0, 1].set_yticks(ax[0, 0].get_yticks())
ax[0, 1].set_ylim(ax[0, 0].get_ylim())
ax[0, 1].set_title('Prediction distribution')
ax[0, 2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red'))
ax[0, 2].set_title('Test set summary statistics')
ax[0, 2].set_xticks([])
ax[0, 3].boxplot(y_pred.ravel())
ax[0, 3].set_title('Predictions summary statistics')
ax[0, 3].set_xticks([])
ax[0, 3].set_yticks(ax[0, 2].get_yticks())
ax[0, 3].set_ylim(ax[0, 2].get_ylim())
## Log transforms
y_test = y_test.apply(np.log10)
y_pred = np.log10(y_pred)
ax[1, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
ax[1, 0].set_title('Test set distribution (log10)')
ax[1, 1].hist(y_pred.ravel(), bins=25)
ax[1, 1].set_xticks(ax[1, 0].get_xticks())
ax[1, 1].set_xlim(ax[1, 0].get_xlim())
ax[1, 0].set_ylabel('count')
ax[1, 1].set_ylabel('count')
ax[1, 1].set_title('Prediction distribution (log10)')
ax[1, 2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red'))
ax[1, 2].set_title('Test set summary statistics (log10)')
ax[1, 2].set_xticks([])
ax[1, 3].boxplot(y_pred.ravel())
ax[1, 3].set_title('Predictions summary statistics (log10)')
ax[1, 3].set_xticks([])
ax[1, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
ax[1, 3].set_yticks(ax[1, 2].get_yticks())
ax[1, 3].set_ylim(ax[1, 2].get_ylim())
handle_savefig(savefig)
def error_summary(models_to_load,
model_specs,
save_path,
ylimr,
data_dict=FULL_DATA_DICT):
def populate_axes(ax, df, col: str, percentage=False):
bars = sns.barplot(x=df.index, y=df[col], ax=ax, alpha=0.8)
ax.set_xticklabels(df.index, rotation=25, ha='right')
for p in bars.patches:
label = format(p.get_height(),
'.2f') + '%' if percentage else format(
int(p.get_height()), ',d')
bars.annotate(label,
(p.get_x() + p.get_width() / 2., p.get_height()),
ha='center',
va='center',
xytext=(0, 10),
textcoords='offset points')
X_train, X_test, y_train, y_test = DataManager.load_tts_data(**data_dict)
scores = [
compute_scores(model_path, X_test, y_test)
for model_path in models_to_load
]
df = pd.DataFrame(scores)
df.columns = ['Model', 'MAE', 'MAPE Transformed', 'MAPE']
df.set_index('Model', inplace=True)
sns.set_style(style='whitegrid')
fig, (ax1, ax2, ax3) = plt.subplots(1,
3,
figsize=(16, 8),
gridspec_kw={
'top': 0.92,
'bottom': 0.2
})
plt.suptitle("Extended Model Comparison")
populate_axes(ax1, df, 'MAE')
populate_axes(ax2, df, 'MAPE Transformed', percentage=True)
populate_axes(ax3, df, 'MAPE', percentage=True)
plt.tight_layout()
print("Done")
ut.handle_savefig(save_path)
def compute_baseline_for_labels(start=1, end=2, savefig=None, log_tf=False):
knn = KNeighborsRegressor(n_neighbors=1) # Baseline classifier
medians = []
means = []
errors = []
for i in range(start, end + 1):
X_train, X_test, y_train, y_test = DataManager.load_tts_data(cnn_ds=True, cnn_agg=False, ohe_cnn=True,
n_labels=i) # TODO possibly faulty
if log_tf:
y_train = y_train.apply(np.log10)
y_test = y_test.apply(np.log10)
knn.fit(X_train, y_train.iloc[:, 0].ravel())
y_pred = knn.predict(X_test)
median = np.median(y_pred, axis=0)
medians.append(median)
means.append(np.mean(y_pred, axis=0))
m_err = mean_absolute_error(y_test.iloc[:, 0].ravel(), y_pred.ravel())
errors.append(m_err)
print(f'_____________ N_labels: {i} ______________')
print(f'Prediction median: {median}')
print(f'Mean absolute error: {m_err}\n')
_, _, _, y_test = DataManager.load_tts_data(cnn_agg=False, ohe_cnn=True, n_labels=1)
fig = plt.figure(figsize=(14, 8))
plt.title("Baseline predictions for all top-x CNN-labels")
plt.xlabel("Number of stored top predictions per frame")
plt.ylabel("")
plt.plot(np.array(medians), color='yellow', marker='o')
plt.plot(np.array(means), color='blue', marker='x')
plt.plot(np.array(errors), color='red')
plt.plot(np.ones((end - start + 1,), dtype=np.float64) * y_test.mean().values, color='black')
plt.legend(('Median', 'mean', 'm abs err', 'True mean'))
plt.xticks(range(start, end + 1))
handle_savefig(savefig)
def compute_KNN1_baseline(savefig=['baseline', 'baseline_test_pred_comparison.png'], log_tf=True):
X_train, X_test, y_train, y_test = DataManager.load_tts_data(**BL_DATA_DICT)
if log_tf:
y_train = y_train.apply(np.log10)
y_test = y_test.apply(np.log10)
knn = KNeighborsRegressor(n_neighbors=1)
knn.fit(X_train, y_train.iloc[:, 0].ravel())
y_pred = knn.predict(X_test)
print(f'Median of predictions: {np.median(y_pred)}')
m_err = mean_absolute_error(y_test.iloc[:, 0].ravel(), y_pred.ravel())
print(f'Mean absolute error: {m_err}')
r2 = knn.score(X_test, y_test.iloc[:, 0].ravel())
print(f'R^2={r2}')
fig, ax = plt.subplots(1, 4, figsize=(20, 5),
gridspec_kw={'top': 0.9, 'left': 0.1, 'right': 0.92, 'height_ratios': [1]})
plt.suptitle(f'Test/Pred comparison: R^2={r2}, m_abs={m_err}')
ax[0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
ax[0].set(yscale='log')
ax[0].set_title('Test set distribution')
ax[1].hist(y_pred.ravel(), bins=25)
ax[1].set_title('Prediction distribution')
ax[1].set(yscale='log')
ax[2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red'))
ax[2].set_title('Test set summary statistics')
ax[2].set_xticks([])
ax[3].boxplot(y_pred.ravel())
ax[3].set_title('Predictions summary statistics')
ax[3].set_xticks([])
handle_savefig(savefig)
'std_scale': True,
'with_mean': False,
'log_tf': True
}
data_dict = BL_DATA_DICT
models = [
KNeighborsRegressor(),
SVR(),
RandomForestRegressor(),
MLPRegressor()
]
names = [type(model).__name__ for model in models]
fitted = []
errors = []
r2s = []
for model in tqdm(models):
pipeline, m_err, r2 = train_test_model(model,
params,
data_dict,
pp_dict,
save_model=True)
fitted.append(pipeline)
errors.append(m_err)
r2s.append(r2)
fig = plt.figure(figsize=(10, 5))
sns.barplot(x=names[:-1], y=errors[:-1])
handle_savefig(['basic_models', 'error_comparison'])