def compare_models(stats, path_spec, ylimr=(-0.3, 0.25), title="Final Model Comparison"): df = pd.DataFrame.from_dict(stats, orient='index') df.columns = ['mean absolute error', '$R^2$'] sns.set_style(style='whitegrid') fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5), gridspec_kw={'top': 0.92}) plt.suptitle(title) bars = sns.barplot(x=df.index, y=df['mean absolute error'], ax=ax1, alpha=0.8) for p in bars.patches: bars.annotate(format(int(p.get_height()), ',d'), (p.get_x() + p.get_width() / 2., p.get_height()), ha='center', va='center', xytext=(0, 10), textcoords='offset points') ax1.set_ylim((3.5e7, 5e7)) bars = sns.barplot(x=df.index, y=df['$R^2$'], ax=ax2, alpha=0.8) for p in bars.patches: if p.get_height() > 0: bars.annotate(format(p.get_height(), '.4f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha='center', va='center', xytext=(0, 10), textcoords='offset points') else: bars.annotate(format(p.get_height(), '.4f'), (p.get_x() + p.get_width() / 2., p.get_height() - 0.04), ha='center', va='center', xytext=(0, 10), textcoords='offset points') ax2.set_ylim(ylimr) handle_savefig(path_spec)
def plot_predictions(y_true, y_pred, n=10, savefig=None): idx = y_true.ravel().argsort(axis=0) y_true = y_true[idx] y_pred = y_pred[idx] fig = plt.figure(figsize=(10, 10)) plt.scatter(x=y_true[:n], y=y_pred[:n], marker='x', color='black') plt.xlabel('True viewcounts') plt.ylabel('Predicted viewcounts') plt.yscale(plt.gca().get_xscale()) plt.yticks(plt.gca().get_xticks()) plt.ylim(plt.gca().get_xlim()) handle_savefig(savefig)
def plot_sorted_residuals(path, data_dict=FULL_DATA_DICT, model_info=None, savefig=None, log_tf=True): pipeline = ut.load_model(path) X_train, X_test, y_train, y_test = DataManager.load_tts_data(**data_dict) y_pred = pipeline.predict(X_test) if check_log_tf(path): y_pred = DataManager.inv_tf(y_pred) y_test = np.array(y_test).ravel() sorted_idx = y_test.argsort() y_test_srt = y_test[sorted_idx] y_pred_srt = y_pred[sorted_idx] if log_tf: y_test_srt, y_pred_srt = DataManager.log_tf(y_test_srt, y_pred_srt) fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(15, 10), gridspec_kw={'top': 0.90}) x = range(len(y_test_srt)) mape_tf = mean_absolute_percentage_error(y_test_srt, y_pred_srt) mape = mean_absolute_percentage_error(y_test, y_pred) print(mape) plt.suptitle('Sorted Viewcount error comparison:\n' + 'MAPE Transformed: {:.2%}\n'.format(mape_tf / 100) + 'MAPE: {:.2%}\n'.format(mape / 100) + generate_model_descr(pipeline, model_info)) plt.tight_layout() ax1.scatter(x=x, y=y_test_srt, s=1.5, alpha=0.9, marker='x') ax2.scatter(x=x, y=y_pred_srt, s=1.5, alpha=0.9, marker='x') ax2.set_ylabel('predicted log(viewcounts)' if log_tf else 'viewcounts') ax1.set_ylabel('true log(viewcounts)' if log_tf else 'viewcounts') ax2.set_ylim(ax1.get_ylim()) ax2.set_xticks([]) ax1.set_xticks([]) rel_error = np.abs((y_test_srt - y_pred_srt) / y_test_srt) ax3.scatter(x=x, y=rel_error, s=1.5, alpha=0.9, marker='x', c='red') # ax3.set_ylim((0.0, 100.00)) ax3.set_ylabel('Relative Error') ax3.set_xlabel('Sorted Sample Index') ut.handle_savefig(savefig) return mean_absolute_percentage_error(y_test, y_pred)
def plot_prediction_results(y_test, y_pred, loss_function=mean_absolute_error, savefig=None, title='model'): err = loss_function(y_test, y_pred) fig, ax = plt.subplots(2, 4, figsize=(20, 10), gridspec_kw={'hspace': 0.2}) plt.suptitle(f'Test/Pred of {title}: {loss_function.__name__}={err}') ax[0, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red') ax[0, 0].set_title('Test set distribution') ax[0, 1].hist(y_pred.ravel(), bins=25) ax[0, 0].set_yscale('log') ax[0, 1].set_yscale('log') ax[0, 0].set_ylabel('count (log)') ax[0, 1].set_ylabel('count (log)') ax[0, 1].set_yticks(ax[0, 0].get_yticks()) ax[0, 1].set_ylim(ax[0, 0].get_ylim()) ax[0, 1].set_title('Prediction distribution') ax[0, 2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red')) ax[0, 2].set_title('Test set summary statistics') ax[0, 2].set_xticks([]) ax[0, 3].boxplot(y_pred.ravel()) ax[0, 3].set_title('Predictions summary statistics') ax[0, 3].set_xticks([]) ax[0, 3].set_yticks(ax[0, 2].get_yticks()) ax[0, 3].set_ylim(ax[0, 2].get_ylim()) ## Log transforms y_test = y_test.apply(np.log10) y_pred = np.log10(y_pred) ax[1, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red') ax[1, 0].set_title('Test set distribution (log10)') ax[1, 1].hist(y_pred.ravel(), bins=25) ax[1, 1].set_xticks(ax[1, 0].get_xticks()) ax[1, 1].set_xlim(ax[1, 0].get_xlim()) ax[1, 0].set_ylabel('count') ax[1, 1].set_ylabel('count') ax[1, 1].set_title('Prediction distribution (log10)') ax[1, 2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red')) ax[1, 2].set_title('Test set summary statistics (log10)') ax[1, 2].set_xticks([]) ax[1, 3].boxplot(y_pred.ravel()) ax[1, 3].set_title('Predictions summary statistics (log10)') ax[1, 3].set_xticks([]) ax[1, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red') ax[1, 3].set_yticks(ax[1, 2].get_yticks()) ax[1, 3].set_ylim(ax[1, 2].get_ylim()) handle_savefig(savefig)
def error_summary(models_to_load, model_specs, save_path, ylimr, data_dict=FULL_DATA_DICT): def populate_axes(ax, df, col: str, percentage=False): bars = sns.barplot(x=df.index, y=df[col], ax=ax, alpha=0.8) ax.set_xticklabels(df.index, rotation=25, ha='right') for p in bars.patches: label = format(p.get_height(), '.2f') + '%' if percentage else format( int(p.get_height()), ',d') bars.annotate(label, (p.get_x() + p.get_width() / 2., p.get_height()), ha='center', va='center', xytext=(0, 10), textcoords='offset points') X_train, X_test, y_train, y_test = DataManager.load_tts_data(**data_dict) scores = [ compute_scores(model_path, X_test, y_test) for model_path in models_to_load ] df = pd.DataFrame(scores) df.columns = ['Model', 'MAE', 'MAPE Transformed', 'MAPE'] df.set_index('Model', inplace=True) sns.set_style(style='whitegrid') fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(16, 8), gridspec_kw={ 'top': 0.92, 'bottom': 0.2 }) plt.suptitle("Extended Model Comparison") populate_axes(ax1, df, 'MAE') populate_axes(ax2, df, 'MAPE Transformed', percentage=True) populate_axes(ax3, df, 'MAPE', percentage=True) plt.tight_layout() print("Done") ut.handle_savefig(save_path)
def compute_baseline_for_labels(start=1, end=2, savefig=None, log_tf=False): knn = KNeighborsRegressor(n_neighbors=1) # Baseline classifier medians = [] means = [] errors = [] for i in range(start, end + 1): X_train, X_test, y_train, y_test = DataManager.load_tts_data(cnn_ds=True, cnn_agg=False, ohe_cnn=True, n_labels=i) # TODO possibly faulty if log_tf: y_train = y_train.apply(np.log10) y_test = y_test.apply(np.log10) knn.fit(X_train, y_train.iloc[:, 0].ravel()) y_pred = knn.predict(X_test) median = np.median(y_pred, axis=0) medians.append(median) means.append(np.mean(y_pred, axis=0)) m_err = mean_absolute_error(y_test.iloc[:, 0].ravel(), y_pred.ravel()) errors.append(m_err) print(f'_____________ N_labels: {i} ______________') print(f'Prediction median: {median}') print(f'Mean absolute error: {m_err}\n') _, _, _, y_test = DataManager.load_tts_data(cnn_agg=False, ohe_cnn=True, n_labels=1) fig = plt.figure(figsize=(14, 8)) plt.title("Baseline predictions for all top-x CNN-labels") plt.xlabel("Number of stored top predictions per frame") plt.ylabel("") plt.plot(np.array(medians), color='yellow', marker='o') plt.plot(np.array(means), color='blue', marker='x') plt.plot(np.array(errors), color='red') plt.plot(np.ones((end - start + 1,), dtype=np.float64) * y_test.mean().values, color='black') plt.legend(('Median', 'mean', 'm abs err', 'True mean')) plt.xticks(range(start, end + 1)) handle_savefig(savefig)
def compute_KNN1_baseline(savefig=['baseline', 'baseline_test_pred_comparison.png'], log_tf=True): X_train, X_test, y_train, y_test = DataManager.load_tts_data(**BL_DATA_DICT) if log_tf: y_train = y_train.apply(np.log10) y_test = y_test.apply(np.log10) knn = KNeighborsRegressor(n_neighbors=1) knn.fit(X_train, y_train.iloc[:, 0].ravel()) y_pred = knn.predict(X_test) print(f'Median of predictions: {np.median(y_pred)}') m_err = mean_absolute_error(y_test.iloc[:, 0].ravel(), y_pred.ravel()) print(f'Mean absolute error: {m_err}') r2 = knn.score(X_test, y_test.iloc[:, 0].ravel()) print(f'R^2={r2}') fig, ax = plt.subplots(1, 4, figsize=(20, 5), gridspec_kw={'top': 0.9, 'left': 0.1, 'right': 0.92, 'height_ratios': [1]}) plt.suptitle(f'Test/Pred comparison: R^2={r2}, m_abs={m_err}') ax[0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red') ax[0].set(yscale='log') ax[0].set_title('Test set distribution') ax[1].hist(y_pred.ravel(), bins=25) ax[1].set_title('Prediction distribution') ax[1].set(yscale='log') ax[2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red')) ax[2].set_title('Test set summary statistics') ax[2].set_xticks([]) ax[3].boxplot(y_pred.ravel()) ax[3].set_title('Predictions summary statistics') ax[3].set_xticks([]) handle_savefig(savefig)
'std_scale': True, 'with_mean': False, 'log_tf': True } data_dict = BL_DATA_DICT models = [ KNeighborsRegressor(), SVR(), RandomForestRegressor(), MLPRegressor() ] names = [type(model).__name__ for model in models] fitted = [] errors = [] r2s = [] for model in tqdm(models): pipeline, m_err, r2 = train_test_model(model, params, data_dict, pp_dict, save_model=True) fitted.append(pipeline) errors.append(m_err) r2s.append(r2) fig = plt.figure(figsize=(10, 5)) sns.barplot(x=names[:-1], y=errors[:-1]) handle_savefig(['basic_models', 'error_comparison'])