Esempio n. 1
0
def compare_models(stats, path_spec, ylimr=(-0.3, 0.25), title="Final Model Comparison"):
    df = pd.DataFrame.from_dict(stats, orient='index')
    df.columns = ['mean absolute error', '$R^2$']

    sns.set_style(style='whitegrid')
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5), gridspec_kw={'top': 0.92})

    plt.suptitle(title)
    bars = sns.barplot(x=df.index, y=df['mean absolute error'], ax=ax1, alpha=0.8)
    for p in bars.patches:
        bars.annotate(format(int(p.get_height()), ',d'), (p.get_x() + p.get_width() / 2., p.get_height()), ha='center',
                      va='center', xytext=(0, 10), textcoords='offset points')
    ax1.set_ylim((3.5e7, 5e7))
    bars = sns.barplot(x=df.index, y=df['$R^2$'], ax=ax2, alpha=0.8)
    for p in bars.patches:
        if p.get_height() > 0:
            bars.annotate(format(p.get_height(), '.4f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha='center',
                          va='center', xytext=(0, 10), textcoords='offset points')
        else:
            bars.annotate(format(p.get_height(), '.4f'), (p.get_x() + p.get_width() / 2., p.get_height() - 0.04),
                          ha='center',
                          va='center', xytext=(0, 10), textcoords='offset points')
    ax2.set_ylim(ylimr)

    handle_savefig(path_spec)
Esempio n. 2
0
def plot_predictions(y_true, y_pred, n=10, savefig=None):
    idx = y_true.ravel().argsort(axis=0)
    y_true = y_true[idx]
    y_pred = y_pred[idx]
    fig = plt.figure(figsize=(10, 10))
    plt.scatter(x=y_true[:n], y=y_pred[:n], marker='x', color='black')
    plt.xlabel('True viewcounts')
    plt.ylabel('Predicted viewcounts')
    plt.yscale(plt.gca().get_xscale())
    plt.yticks(plt.gca().get_xticks())
    plt.ylim(plt.gca().get_xlim())

    handle_savefig(savefig)
Esempio n. 3
0
def plot_sorted_residuals(path,
                          data_dict=FULL_DATA_DICT,
                          model_info=None,
                          savefig=None,
                          log_tf=True):
    pipeline = ut.load_model(path)
    X_train, X_test, y_train, y_test = DataManager.load_tts_data(**data_dict)
    y_pred = pipeline.predict(X_test)
    if check_log_tf(path):
        y_pred = DataManager.inv_tf(y_pred)

    y_test = np.array(y_test).ravel()

    sorted_idx = y_test.argsort()
    y_test_srt = y_test[sorted_idx]
    y_pred_srt = y_pred[sorted_idx]

    if log_tf:
        y_test_srt, y_pred_srt = DataManager.log_tf(y_test_srt, y_pred_srt)

    fig, (ax1, ax2, ax3) = plt.subplots(3,
                                        1,
                                        figsize=(15, 10),
                                        gridspec_kw={'top': 0.90})

    x = range(len(y_test_srt))
    mape_tf = mean_absolute_percentage_error(y_test_srt, y_pred_srt)
    mape = mean_absolute_percentage_error(y_test, y_pred)
    print(mape)
    plt.suptitle('Sorted Viewcount error comparison:\n' +
                 'MAPE Transformed: {:.2%}\n'.format(mape_tf / 100) +
                 'MAPE: {:.2%}\n'.format(mape / 100) +
                 generate_model_descr(pipeline, model_info))
    plt.tight_layout()
    ax1.scatter(x=x, y=y_test_srt, s=1.5, alpha=0.9, marker='x')
    ax2.scatter(x=x, y=y_pred_srt, s=1.5, alpha=0.9, marker='x')
    ax2.set_ylabel('predicted log(viewcounts)' if log_tf else 'viewcounts')
    ax1.set_ylabel('true log(viewcounts)' if log_tf else 'viewcounts')
    ax2.set_ylim(ax1.get_ylim())
    ax2.set_xticks([])
    ax1.set_xticks([])

    rel_error = np.abs((y_test_srt - y_pred_srt) / y_test_srt)
    ax3.scatter(x=x, y=rel_error, s=1.5, alpha=0.9, marker='x', c='red')
    # ax3.set_ylim((0.0, 100.00))
    ax3.set_ylabel('Relative Error')
    ax3.set_xlabel('Sorted Sample Index')
    ut.handle_savefig(savefig)

    return mean_absolute_percentage_error(y_test, y_pred)
Esempio n. 4
0
def plot_prediction_results(y_test, y_pred, loss_function=mean_absolute_error, savefig=None, title='model'):
    err = loss_function(y_test, y_pred)

    fig, ax = plt.subplots(2, 4, figsize=(20, 10), gridspec_kw={'hspace': 0.2})
    plt.suptitle(f'Test/Pred of {title}: {loss_function.__name__}={err}')
    ax[0, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
    ax[0, 0].set_title('Test set distribution')
    ax[0, 1].hist(y_pred.ravel(), bins=25)
    ax[0, 0].set_yscale('log')
    ax[0, 1].set_yscale('log')
    ax[0, 0].set_ylabel('count (log)')
    ax[0, 1].set_ylabel('count (log)')
    ax[0, 1].set_yticks(ax[0, 0].get_yticks())
    ax[0, 1].set_ylim(ax[0, 0].get_ylim())

    ax[0, 1].set_title('Prediction distribution')
    ax[0, 2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red'))
    ax[0, 2].set_title('Test set summary statistics')
    ax[0, 2].set_xticks([])
    ax[0, 3].boxplot(y_pred.ravel())
    ax[0, 3].set_title('Predictions summary statistics')
    ax[0, 3].set_xticks([])
    ax[0, 3].set_yticks(ax[0, 2].get_yticks())
    ax[0, 3].set_ylim(ax[0, 2].get_ylim())

    ## Log transforms
    y_test = y_test.apply(np.log10)
    y_pred = np.log10(y_pred)
    ax[1, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
    ax[1, 0].set_title('Test set distribution (log10)')
    ax[1, 1].hist(y_pred.ravel(), bins=25)
    ax[1, 1].set_xticks(ax[1, 0].get_xticks())
    ax[1, 1].set_xlim(ax[1, 0].get_xlim())
    ax[1, 0].set_ylabel('count')
    ax[1, 1].set_ylabel('count')

    ax[1, 1].set_title('Prediction distribution (log10)')
    ax[1, 2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red'))
    ax[1, 2].set_title('Test set summary statistics (log10)')
    ax[1, 2].set_xticks([])
    ax[1, 3].boxplot(y_pred.ravel())
    ax[1, 3].set_title('Predictions summary statistics (log10)')
    ax[1, 3].set_xticks([])
    ax[1, 0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
    ax[1, 3].set_yticks(ax[1, 2].get_yticks())
    ax[1, 3].set_ylim(ax[1, 2].get_ylim())

    handle_savefig(savefig)
Esempio n. 5
0
def error_summary(models_to_load,
                  model_specs,
                  save_path,
                  ylimr,
                  data_dict=FULL_DATA_DICT):
    def populate_axes(ax, df, col: str, percentage=False):
        bars = sns.barplot(x=df.index, y=df[col], ax=ax, alpha=0.8)
        ax.set_xticklabels(df.index, rotation=25, ha='right')
        for p in bars.patches:
            label = format(p.get_height(),
                           '.2f') + '%' if percentage else format(
                               int(p.get_height()), ',d')
            bars.annotate(label,
                          (p.get_x() + p.get_width() / 2., p.get_height()),
                          ha='center',
                          va='center',
                          xytext=(0, 10),
                          textcoords='offset points')

    X_train, X_test, y_train, y_test = DataManager.load_tts_data(**data_dict)
    scores = [
        compute_scores(model_path, X_test, y_test)
        for model_path in models_to_load
    ]

    df = pd.DataFrame(scores)
    df.columns = ['Model', 'MAE', 'MAPE Transformed', 'MAPE']
    df.set_index('Model', inplace=True)

    sns.set_style(style='whitegrid')
    fig, (ax1, ax2, ax3) = plt.subplots(1,
                                        3,
                                        figsize=(16, 8),
                                        gridspec_kw={
                                            'top': 0.92,
                                            'bottom': 0.2
                                        })

    plt.suptitle("Extended Model Comparison")
    populate_axes(ax1, df, 'MAE')
    populate_axes(ax2, df, 'MAPE Transformed', percentage=True)
    populate_axes(ax3, df, 'MAPE', percentage=True)
    plt.tight_layout()
    print("Done")
    ut.handle_savefig(save_path)
Esempio n. 6
0
def compute_baseline_for_labels(start=1, end=2, savefig=None, log_tf=False):
    knn = KNeighborsRegressor(n_neighbors=1)  # Baseline classifier

    medians = []
    means = []
    errors = []

    for i in range(start, end + 1):
        X_train, X_test, y_train, y_test = DataManager.load_tts_data(cnn_ds=True, cnn_agg=False, ohe_cnn=True,
                                                                     n_labels=i)  # TODO possibly faulty

        if log_tf:
            y_train = y_train.apply(np.log10)
            y_test = y_test.apply(np.log10)

        knn.fit(X_train, y_train.iloc[:, 0].ravel())

        y_pred = knn.predict(X_test)

        median = np.median(y_pred, axis=0)
        medians.append(median)
        means.append(np.mean(y_pred, axis=0))
        m_err = mean_absolute_error(y_test.iloc[:, 0].ravel(), y_pred.ravel())
        errors.append(m_err)
        print(f'_____________  N_labels: {i}  ______________')
        print(f'Prediction median: {median}')
        print(f'Mean absolute error: {m_err}\n')

    _, _, _, y_test = DataManager.load_tts_data(cnn_agg=False, ohe_cnn=True, n_labels=1)
    fig = plt.figure(figsize=(14, 8))
    plt.title("Baseline predictions for all top-x CNN-labels")
    plt.xlabel("Number of stored top predictions per frame")
    plt.ylabel("")
    plt.plot(np.array(medians), color='yellow', marker='o')
    plt.plot(np.array(means), color='blue', marker='x')
    plt.plot(np.array(errors), color='red')
    plt.plot(np.ones((end - start + 1,), dtype=np.float64) * y_test.mean().values, color='black')
    plt.legend(('Median', 'mean', 'm abs err', 'True mean'))
    plt.xticks(range(start, end + 1))
    handle_savefig(savefig)
Esempio n. 7
0
def compute_KNN1_baseline(savefig=['baseline', 'baseline_test_pred_comparison.png'], log_tf=True):
    X_train, X_test, y_train, y_test = DataManager.load_tts_data(**BL_DATA_DICT)

    if log_tf:
        y_train = y_train.apply(np.log10)
        y_test = y_test.apply(np.log10)

    knn = KNeighborsRegressor(n_neighbors=1)

    knn.fit(X_train, y_train.iloc[:, 0].ravel())

    y_pred = knn.predict(X_test)

    print(f'Median of predictions: {np.median(y_pred)}')

    m_err = mean_absolute_error(y_test.iloc[:, 0].ravel(), y_pred.ravel())

    print(f'Mean absolute error: {m_err}')
    r2 = knn.score(X_test, y_test.iloc[:, 0].ravel())
    print(f'R^2={r2}')

    fig, ax = plt.subplots(1, 4, figsize=(20, 5),
                           gridspec_kw={'top': 0.9, 'left': 0.1, 'right': 0.92, 'height_ratios': [1]})
    plt.suptitle(f'Test/Pred comparison: R^2={r2}, m_abs={m_err}')
    ax[0].hist(y_test.iloc[:, 0].ravel(), bins=25, color='red')
    ax[0].set(yscale='log')
    ax[0].set_title('Test set distribution')
    ax[1].hist(y_pred.ravel(), bins=25)
    ax[1].set_title('Prediction distribution')
    ax[1].set(yscale='log')
    ax[2].boxplot(y_test.iloc[:, 0].ravel(), boxprops=dict(color='red'))
    ax[2].set_title('Test set summary statistics')
    ax[2].set_xticks([])
    ax[3].boxplot(y_pred.ravel())
    ax[3].set_title('Predictions summary statistics')
    ax[3].set_xticks([])
    handle_savefig(savefig)
    'std_scale': True,
    'with_mean': False,
    'log_tf': True
}
data_dict = BL_DATA_DICT

models = [
    KNeighborsRegressor(),
    SVR(),
    RandomForestRegressor(),
    MLPRegressor()
]
names = [type(model).__name__ for model in models]
fitted = []
errors = []
r2s = []

for model in tqdm(models):
    pipeline, m_err, r2 = train_test_model(model,
                                           params,
                                           data_dict,
                                           pp_dict,
                                           save_model=True)
    fitted.append(pipeline)
    errors.append(m_err)
    r2s.append(r2)

fig = plt.figure(figsize=(10, 5))
sns.barplot(x=names[:-1], y=errors[:-1])
handle_savefig(['basic_models', 'error_comparison'])