Esempio n. 1
0
 def smooth_valacc(self, step=None):
     if step is None:
         step = self.print_every
     return smooth(self.val_accs, step=step)
Esempio n. 2
0
 def smooth_loss(self, step=None):
     if step is None:
         step = self.print_every
     return smooth(self.all_losses, step=step)
Esempio n. 3
0
def plot_best(pattern,
              smooth_window=1,
              title=None,
              ylim=None,
              xlim=None,
              methods=None,
              method2label=None,
              desired_settings=None,
              report_result=False,
              start=0,
              xlabel=None,
              ylabel=None,
              end=-1,
              default_setting=None,
              q_low=25,
              q_high=75):
    import matplotlib.pyplot as plt
    import seaborn as sns
    sns.set_style("whitegrid")
    methods_settings = {}
    accs = {}
    for fn in sorted(glob.glob(pattern)):
        method_setting = fn.split('/')[-1].split('^')[0].split('-')
        method, setting = method_setting[0], '-'.join(method_setting[1:])

        if desired_settings is not None and setting not in desired_settings:
            continue
        if methods is not None and method not in methods:
            continue

        if method not in methods_settings:
            methods_settings[method] = {}
            accs[method] = {}
        if setting not in methods_settings[method]:
            methods_settings[method][setting] = []
            accs[method][setting] = []

        tr_loss = smooth(joblib.load(fn), smooth_window)

        methods_settings[method][setting].append(tr_loss)
        accs[method][setting].append(float(fn.split('/')[-1].split('^')[1]))

    if len(methods_settings.keys()
           ) > 1:  # plot the best one, smallest area under the curve
        for method, setting_dict in methods_settings.items():
            setting_areas = []
            for setting, v in setting_dict.items():
                # find the smallest area
                max_len = max([len(a) for a in v])
                v = [a for a in v if len(a) == max_len]
                #min_len = min([len(a) for a in v])
                #end_ = min_len if end == -1 else min(end, min_len)
                #area = np.mean([a[start:end_] for a in v])
                area = np.mean([a[start:end] for a in v])
                setting_areas.append((setting, area))
            setting = sorted(setting_areas, key=lambda x: x[1])[0][0]

            v = setting_dict[setting]
            max_len = max([len(a) for a in v])
            v = [a for a in v if len(a) == max_len]

            print('{}: {:.2f}% ({:.2f}) {} runs'.format(
                method, np.mean(accs[method][setting]),
                np.std(accs[method][setting]), len(v)))

            if method2label is None:
                label = method + '-' + setting
            else:
                label = method2label.get(method, method)
            #p = plt.plot(sum(v) / len(v), label=method + '-' + setting)#, c=colors[i])
            p = plt.plot(np.percentile(v, 50, 0), label=label, ls='--')
            plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
                             np.percentile(v, q_low, 0),
                             np.percentile(v, q_high, 0),
                             alpha=0.1,
                             color=p[-1].get_color())

            if default_setting and method in default_setting and default_setting[
                    method] in setting_dict:
                v = setting_dict[default_setting[method]]
                max_len = max([len(a) for a in v])
                v = [a for a in v if len(a) == max_len]

                p = plt.plot(np.percentile(v, 50, 0),
                             ls='-',
                             c=p[-1].get_color())
                plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
                                 np.percentile(v, q_low, 0),
                                 np.percentile(v, q_high, 0),
                                 alpha=0.1,
                                 color=p[-1].get_color())

    else:  # plot all
        for method, setting_dict in methods_settings.items():
            for setting, v in setting_dict.items():
                max_len = max([len(a) for a in v])
                v = [a for a in v if len(a) == max_len]

                print('{}: {:.2f}% ({:.2f}) {} runs'.format(
                    method, np.mean(accs[method][setting]),
                    np.std(accs[method][setting]), len(v)))

                if method2label is None:
                    label = method + '-' + setting
                else:
                    label = method2label.get(method, method) + '-' + setting

                p = plt.plot(np.percentile(v, 50, 0), label=label)
                plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
                                 np.percentile(v, q_low, 0),
                                 np.percentile(v, q_high, 0),
                                 alpha=0.1,
                                 color=p[-1].get_color())

    plt.legend()
    if title:
        plt.title(title, fontsize=15)
    plt.ylim(ylim)
    plt.xlim(xlim)
    ylabel = ylabel or pattern.split("*")[-1]
    xlabel = xlabel or "Epoch"
    plt.ylabel(ylabel, fontsize=15)
    plt.xlabel(xlabel, fontsize=15)
    plt.grid(color='gray', alpha=0.5)
    plt.show()
 def smooth_valloss(self, step=None):
     if step is None:
         step = self.print_every
     return smooth(np.sqrt(self.val_losses), step=step)
Esempio n. 5
0
def plot_best_test(train_pattern,
                   smooth_window=1,
                   title=None,
                   ylim=None,
                   xlim=None,
                   methods=None,
                   method2label=None,
                   desired_settings=None,
                   start=0,
                   end=-1,
                   default_setting=None,
                   ylabel=None,
                   xlabel=None,
                   q_low=25,
                   q_high=75,
                   early_stop=False,
                   early_stop_patience=None):
    import matplotlib.pyplot as plt
    import seaborn as sns
    sns.set_style("whitegrid")
    assert 'train_losses' in train_pattern, 'train_losses must appear in train_pattern'
    methods_settings = {}
    val_settings = {}
    test_settings = {}
    accs = {}  # accs are based on best validation performance
    best_val_test_error = {}
    for fn in sorted(glob.glob(train_pattern)):
        method_setting = fn.split('/')[-1].split('^')[0].split('-')
        method, setting = method_setting[0], '-'.join(method_setting[1:])

        if desired_settings is not None and setting not in desired_settings:
            continue
        if methods is not None and method not in methods:
            continue

        if method not in methods_settings:
            methods_settings[method] = {}
            val_settings[method] = {}
            test_settings[method] = {}
            accs[method] = {}
        if setting not in methods_settings[method]:
            methods_settings[method][setting] = []
            val_settings[method][setting] = []
            test_settings[method][setting] = []
            accs[method][setting] = []

        tr_loss = smooth(joblib.load(fn), smooth_window)

        val_error = joblib.load(fn.replace('train_losses', 'val_errors'))
        test_error = joblib.load(fn.replace('train_losses', 'test_errors'))

        if early_stop:
            index_list = early_stop_curve(val_error, early_stop_patience)
            val_error = np.array(val_error)[index_list]
            test_error = np.array(test_error)[index_list]

        val_error = smooth(val_error, smooth_window)
        test_error = smooth(test_error, smooth_window)

        methods_settings[method][setting].append(tr_loss)
        val_settings[method][setting].append(val_error)
        test_settings[method][setting].append(test_error)

        # change the following two to see different evaluation
        best_error_index = np.argmin(val_error)
        # best_error_index = -1
        min_val_error = val_error[best_error_index]

        accs[method][setting].append(test_error[best_error_index])

        if best_val_test_error.get(method) is None:
            best_val_test_error[method] = (min_val_error,
                                           test_error[best_error_index], fn)
        else:
            val_best_error, test_best_error, fn_best = best_val_test_error[
                method]
            if val_best_error >= min_val_error:
                val_best_error = min_val_error
                test_best_error = test_error[best_error_index]
                best_val_test_error[method] = (val_best_error, test_best_error,
                                               fn)

    for method, setting_dict in val_settings.items():

        val_best_error, test_best_error, fn_best = best_val_test_error[method]
        print('{} test: {:.3f} val: {:.3f} fn:{}'.format(
            method, test_best_error, val_best_error, fn_best))

        setting_areas = []
        for setting, v in setting_dict.items():
            # find the smallest area
            max_len = max([len(a) for a in v])
            v = [a for a in v if len(a) == max_len]
            area = np.mean([a[start:end] for a in v])
            setting_areas.append((setting, area))
        setting = sorted(setting_areas, key=lambda x: x[1])[0][0]

        def plot_setting():
            v = test_settings[method][setting]
            max_len = max([len(a) for a in v])
            v = [a for a in v if len(a) == max_len]

            # print('{}: {:.3f} ({:.2f}) {} runs'.format(method,
            #                                            np.mean(accs[method][setting]),
            #                                            np.std(accs[method][setting]),
            #                                            len(v)))

            if method2label is None:
                label = method + '-' + setting
            else:
                label = method2label.get(method, method)
            #p = plt.plot(sum(v) / len(v), label=method + '-' + setting)#, c=colors[i]
            p = plt.plot(np.percentile(v, 50, 0), label=label, ls='--')
            plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
                             np.percentile(v, q_low, 0),
                             np.percentile(v, q_high, 0),
                             alpha=0.1,
                             color=p[-1].get_color())

            if default_setting and method in default_setting and\
            default_setting[method] in test_settings[method]:
                v = test_settings[method][default_setting[method]]
                max_len = max([len(a) for a in v])
                v = [a for a in v if len(a) == max_len]

                p = plt.plot(np.percentile(v, 50, 0),
                             ls='-',
                             c=p[-1].get_color())
                plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
                                 np.percentile(v, q_low, 0),
                                 np.percentile(v, q_high, 0),
                                 alpha=0.1,
                                 color=p[-1].get_color())

        plot_setting()

    plt.legend()
    if title:
        plt.title(title, fontsize=15)
    plt.ylim(ylim)
    plt.xlim(xlim)
    ylabel = ylabel or "error"
    xlabel = xlabel or "Epoch"
    plt.ylabel('test {}'.format(ylabel), fontsize=15)
    plt.xlabel(xlabel, fontsize=15)
    plt.grid(color='gray', alpha=0.5)
    plt.show()