コード例 #1
0
def record_experiment(identifier='%T-%N', name = 'unnamed', info = '', print_to_console = True, show_figs = None,
            save_figs = True, saved_figure_ext = '.pdf', use_temp_dir = False):
    """
    :param identifier: The string that uniquely identifies this experiment record.  Convention is that it should be in
        the format
    :param name: Base-name of the experiment
    :param print_to_console: If True, print statements still go to console - if False, they're just rerouted to file.
    :param show_figs: Show figures when the experiment produces them.  Can be:
        'hang': Show and hang
        'draw': Show but keep on going
        False: Don't show figures
    """
    # Note: matplotlib imports are internal in order to avoid trouble for people who may import this module without having
    # a working matplotlib (which can occasionally be tricky to install).

    identifier = format_filename(file_string = identifier, base_name=name, current_time = datetime.now())

    if show_figs is None:
        show_figs = 'draw' if is_test_mode() else 'hang'

    assert show_figs in ('hang', 'draw', False)

    if use_temp_dir:
        experiment_directory = tempfile.mkdtemp()
        atexit.register(lambda: shutil.rmtree(experiment_directory))
    else:
        experiment_directory = get_local_path('experiments/{identifier}'.format(identifier=identifier))

    make_dir(experiment_directory)
    make_file_dir(experiment_directory)
    log_file_name = os.path.join(experiment_directory, 'output.txt')
    log_capture_context = PrintAndStoreLogger(log_file_path = log_file_name, print_to_console = print_to_console)
    log_capture_context.__enter__()
    from artemis.plotting.manage_plotting import WhatToDoOnShow
    blocking_show_context = WhatToDoOnShow(show_figs)
    blocking_show_context.__enter__()
    if save_figs:
        from artemis.plotting.saving_plots import SaveFiguresOnShow
        figure_save_context = SaveFiguresOnShow(path = os.path.join(experiment_directory, 'fig-%T-%L'+saved_figure_ext))
        figure_save_context.__enter__()

    _register_current_experiment(name, identifier)

    global _CURRENT_EXPERIMENT_RECORD
    _CURRENT_EXPERIMENT_RECORD = ExperimentRecord(experiment_directory)
    _CURRENT_EXPERIMENT_RECORD.add_info('Name: %s' % (name, ))
    _CURRENT_EXPERIMENT_RECORD.add_info('Identifier: %s' % (identifier, ))
    _CURRENT_EXPERIMENT_RECORD.add_info('Directory: %s' % (_CURRENT_EXPERIMENT_RECORD.get_dir(), ))
    yield _CURRENT_EXPERIMENT_RECORD
    _CURRENT_EXPERIMENT_RECORD = None

    blocking_show_context.__exit__(None, None, None)
    log_capture_context.__exit__(None, None, None)
    if save_figs:
        figure_save_context.__exit__(None, None, None)

    _deregister_current_experiment()
コード例 #2
0
def get_experiment_dir():
    path = os.path.expanduser(
        get_artemis_config_value(
            section="experiments",
            option="experiment_directory",
            write_default=True,
            default_generator=lambda: get_artemis_data_path('experiments')))
    if not os.path.exists(path):
        make_dir(path)
    return path
コード例 #3
0
def archive_record(record):
    """
    :param ExperimentRecord record:
    :return str: New directory
    """
    record_dir = record.get_dir()
    exp_dir, record_name = os.path.split(record_dir)
    new_home = os.path.normpath(os.path.join(exp_dir, '..', 'experiment-archive'))
    if not os.path.exists(new_home):
        make_dir(new_home)
    shutil.move(record_dir, new_home)
    new_record_path = os.path.join(new_home, record_name)
    assert os.path.exists(new_record_path)
    return new_record_path
コード例 #4
0
def record_experiment(identifier='%T-%N', name='unnamed', print_to_console=True, show_figs=None,
                      save_figs=True, saved_figure_ext='.fig.pkl', use_temp_dir=False, date=None, prefix=None):
    """
    :param identifier: The string that uniquely identifies this experiment record.  Convention is that it should be in
        the format
    :param name: Base-name of the experiment
    :param print_to_console: If True, print statements still go to console - if False, they're just rerouted to file.
    :param show_figs: Show figures when the experiment produces them.  Can be:
        'hang': Show and hang
        'draw': Show but keep on going
        False: Don't show figures
    """
    # Note: matplotlib imports are internal in order to avoid trouble for people who may import this module without having
    # a working matplotlib (which can occasionally be tricky to install).
    if date is None:
        date = datetime.now()
    identifier = format_filename(file_string=identifier, base_name=name, current_time=date)

    if show_figs is None:
        show_figs = 'draw' if is_test_mode() else 'hang'

    assert show_figs in ('hang', 'draw', False)

    if use_temp_dir:
        experiment_directory = tempfile.mkdtemp()
        atexit.register(lambda: shutil.rmtree(experiment_directory))
    else:
        experiment_directory = get_local_experiment_path(identifier)

    make_dir(experiment_directory)
    this_record = ExperimentRecord(experiment_directory)

    # Create context that sets the current experiment record
    # and the context which captures stdout (print statements) and logs them.
    contexts = [
        hold_current_experiment_record(this_record),
        CaptureStdOut(log_file_path=os.path.join(experiment_directory, 'output.txt'), print_to_console=print_to_console, prefix=prefix)
        ]

    if is_matplotlib_imported():
        from artemis.plotting.manage_plotting import WhatToDoOnShow
        # Add context that modifies how matplotlib figures are shown.
        contexts.append(WhatToDoOnShow(show_figs))
        if save_figs:
            from artemis.plotting.saving_plots import SaveFiguresOnShow
            # Add context that saves figures when show is called.
            contexts.append(SaveFiguresOnShow(path=os.path.join(experiment_directory, 'fig-%T-%L' + saved_figure_ext)))

    with nested(*contexts):
        yield this_record
コード例 #5
0
def archive_record(record):
    """
    :param ExperimentRecord record:
    :return str: New directory
    """
    record_dir = record.get_dir()
    exp_dir, record_name = os.path.split(record_dir)
    new_home = os.path.normpath(
        os.path.join(exp_dir, '..', 'experiment-archive'))
    if not os.path.exists(new_home):
        make_dir(new_home)
    shutil.move(record_dir, new_home)
    new_record_path = os.path.join(new_home, record_name)
    assert os.path.exists(new_record_path)
    return new_record_path
コード例 #6
0
def demo_optimize_conv_scales(n_epochs=5,
                              comp_weight=1e-11,
                              learning_rate=0.1,
                              error_loss='KL',
                              use_softmax=True,
                              optimizer='sgd',
                              shuffle_training=False):
    """
    Run the scale optimization routine on a convnet.  
    :param n_epochs:
    :param comp_weight:
    :param learning_rate:
    :param error_loss:
    :param use_softmax:
    :param optimizer:
    :param shuffle_training:
    :return:
    """
    if error_loss == 'KL' and not use_softmax:
        raise Exception(
            "It's very strange that you want to use a KL divergence on something other than a softmax error.  I assume you've made a mistake."
        )

    training_videos, training_vgg_inputs = get_vgg_video_splice(
        ['ILSVRC2015_train_00033010', 'ILSVRC2015_train_00336001'],
        shuffle=shuffle_training,
        shuffling_rng=1234)
    test_videos, test_vgg_inputs = get_vgg_video_splice(
        ['ILSVRC2015_train_00033009', 'ILSVRC2015_train_00033007'])

    set_dbplot_figure_size(12, 6)

    n_frames_to_show = 10
    display_frames = np.arange(
        len(test_videos) / n_frames_to_show / 2, len(test_videos),
        len(test_videos) / n_frames_to_show)
    ax1 = dbplot(np.concatenate(test_videos[display_frames], axis=1),
                 "Test Videos",
                 title='',
                 plot_type='pic')
    plt.subplots_adjust(wspace=0, hspace=.05)
    ax1.set_xticks(224 * np.arange(len(display_frames) / 2) * 2 + 224 / 2)
    ax1.tick_params(labelbottom='on')

    layers = get_vgg_layer_specifiers(
        up_to_layer='prob' if use_softmax else 'fc8')

    # Setup the true VGGnet and get the outputs
    f_true = ConvNet.from_init(layers, input_shape=(3, 224, 224)).compile()
    true_test_out = flatten2(
        np.concatenate([
            f_true(frame_positions[None])
            for frame_positions in test_vgg_inputs
        ]))
    top5_true_guesses = argtopk(true_test_out, 5)
    true_guesses = np.argmax(true_test_out, axis=1)
    true_labels = [
        get_vgg_label_at(g, short=True)
        for g in true_guesses[display_frames[::2]]
    ]
    full_convnet_cost = np.array([
        get_full_convnet_computational_cost(layer_specs=layers,
                                            input_shape=(3, 224, 224))
    ] * len(test_videos))

    # Setup the approximate networks
    slrc_net = ScaleLearningRoundingConvnet.from_convnet_specs(
        layers,
        optimizer=get_named_optimizer(optimizer, learning_rate=learning_rate),
        corruption_type='rand',
        rng=1234)
    f_train_slrc = slrc_net.train_scales.partial(
        comp_weight=comp_weight, error_loss=error_loss).compile()
    f_get_scales = slrc_net.get_scales.compile()
    round_fp = RoundConvNetForwardPass(layers)
    sigmadelta_fp = SigmaDeltaConvNetForwardPass(layers,
                                                 input_shape=(3, 224, 224))

    p = ProgressIndicator(n_epochs * len(training_videos))

    output_dir = make_dir(get_local_path('output/%T-convnet-spikes'))

    for input_minibatch, minibatch_info in minibatch_iterate_info(
            training_vgg_inputs,
            n_epochs=n_epochs,
            minibatch_size=1,
            test_epochs=np.arange(0, n_epochs, 0.1)):

        if minibatch_info.test_now:
            with EZProfiler('test'):
                current_scales = f_get_scales()
                round_cost, round_out = round_fp.get_cost_and_output(
                    test_vgg_inputs, scales=current_scales)
                sd_cost, sd_out = sigmadelta_fp.get_cost_and_output(
                    test_vgg_inputs, scales=current_scales)
                round_guesses, round_top1_correct, round_top5_correct = get_and_report_scores(
                    round_cost,
                    round_out,
                    name='Round',
                    true_top_1=true_guesses,
                    true_top_k=top5_true_guesses)
                sd_guesses, sd_top1_correct, sd_top5_correct = get_and_report_scores(
                    sd_cost,
                    sd_out,
                    name='SigmaDelta',
                    true_top_1=true_guesses,
                    true_top_k=top5_true_guesses)

                round_labels = [
                    get_vgg_label_at(g, short=True)
                    for g in round_guesses[display_frames[::2]]
                ]

                ax1.set_xticklabels([
                    '{}\n{}'.format(tg, rg)
                    for tg, rg in izip_equal(true_labels, round_labels)
                ])

                ax = dbplot(
                    np.array([
                        round_cost / 1e9, sd_cost / 1e9,
                        full_convnet_cost / 1e9
                    ]).T,
                    'Computation',
                    plot_type='thick-line',
                    ylabel='GOps',
                    title='',
                    legend=['Round', '$\Sigma\Delta$', 'Original'],
                )
                ax.set_xticklabels([])
                plt.grid()
                dbplot(
                    100 * np.array(
                        [cummean(sd_top1_correct),
                         cummean(sd_top5_correct)]).T,
                    "Score",
                    plot_type=lambda: LinePlot(
                        y_bounds=(0, 100),
                        plot_kwargs=[
                            dict(linewidth=3, color='k'),
                            dict(linewidth=3, color='k', linestyle=':')
                        ]),
                    title='',
                    legend=[
                        'Round/$\Sigma\Delta$ Top-1',
                        'Round/$\Sigma\Delta$ Top-5'
                    ],
                    ylabel='Cumulative\nPercent Accuracy',
                    xlabel='Frame #',
                    layout='v',
                )
                plt.grid()
            plt.savefig(
                os.path.join(output_dir,
                             'epoch-%.3g.pdf' % (minibatch_info.epoch, )))
        f_train_slrc(input_minibatch)
        p()
        print "Epoch {:3.2f}: Scales: {}".format(
            minibatch_info.epoch, ['%.3g' % float(s) for s in f_get_scales()])

    results = dict(current_scales=current_scales,
                   round_cost=round_cost,
                   round_out=round_out,
                   sd_cost=sd_cost,
                   sd_out=sd_out,
                   round_guesses=round_guesses,
                   round_top1_correct=round_top1_correct,
                   round_top5_correct=round_top5_correct,
                   sd_guesses=sd_guesses,
                   sd_top1_correct=sd_top1_correct,
                   sd_top5_correct=sd_top5_correct)

    dbplot_hang()
    return results
コード例 #7
0
ファイル: experiment_record.py プロジェクト: QUVA-Lab/artemis
def get_experiment_dir():
    path = os.path.expanduser(get_artemis_config_value(section="experiments", option="experiment_directory", write_default=True, default_generator=lambda: get_artemis_data_path('experiments')))
    if not os.path.exists(path):
        make_dir(path)
    return path