def record_experiment(identifier='%T-%N', name = 'unnamed', info = '', print_to_console = True, show_figs = None,
save_figs = True, saved_figure_ext = '.pdf', use_temp_dir = False):
"""
:param identifier: The string that uniquely identifies this experiment record. Convention is that it should be in
the format
:param name: Base-name of the experiment
:param print_to_console: If True, print statements still go to console - if False, they're just rerouted to file.
:param show_figs: Show figures when the experiment produces them. Can be:
'hang': Show and hang
'draw': Show but keep on going
False: Don't show figures
"""
# Note: matplotlib imports are internal in order to avoid trouble for people who may import this module without having
# a working matplotlib (which can occasionally be tricky to install).
identifier = format_filename(file_string = identifier, base_name=name, current_time = datetime.now())
if show_figs is None:
show_figs = 'draw' if is_test_mode() else 'hang'
assert show_figs in ('hang', 'draw', False)
if use_temp_dir:
experiment_directory = tempfile.mkdtemp()
atexit.register(lambda: shutil.rmtree(experiment_directory))
else:
experiment_directory = get_local_path('experiments/{identifier}'.format(identifier=identifier))
make_dir(experiment_directory)
make_file_dir(experiment_directory)
log_file_name = os.path.join(experiment_directory, 'output.txt')
log_capture_context = PrintAndStoreLogger(log_file_path = log_file_name, print_to_console = print_to_console)
log_capture_context.__enter__()
from artemis.plotting.manage_plotting import WhatToDoOnShow
blocking_show_context = WhatToDoOnShow(show_figs)
blocking_show_context.__enter__()
if save_figs:
from artemis.plotting.saving_plots import SaveFiguresOnShow
figure_save_context = SaveFiguresOnShow(path = os.path.join(experiment_directory, 'fig-%T-%L'+saved_figure_ext))
figure_save_context.__enter__()
_register_current_experiment(name, identifier)
global _CURRENT_EXPERIMENT_RECORD
_CURRENT_EXPERIMENT_RECORD = ExperimentRecord(experiment_directory)
_CURRENT_EXPERIMENT_RECORD.add_info('Name: %s' % (name, ))
_CURRENT_EXPERIMENT_RECORD.add_info('Identifier: %s' % (identifier, ))
_CURRENT_EXPERIMENT_RECORD.add_info('Directory: %s' % (_CURRENT_EXPERIMENT_RECORD.get_dir(), ))
yield _CURRENT_EXPERIMENT_RECORD
_CURRENT_EXPERIMENT_RECORD = None
blocking_show_context.__exit__(None, None, None)
log_capture_context.__exit__(None, None, None)
if save_figs:
figure_save_context.__exit__(None, None, None)
_deregister_current_experiment()
def get_experiment_dir():
path = os.path.expanduser(
get_artemis_config_value(
section="experiments",
option="experiment_directory",
write_default=True,
default_generator=lambda: get_artemis_data_path('experiments')))
if not os.path.exists(path):
make_dir(path)
return path
def archive_record(record):
"""
:param ExperimentRecord record:
:return str: New directory
"""
record_dir = record.get_dir()
exp_dir, record_name = os.path.split(record_dir)
new_home = os.path.normpath(os.path.join(exp_dir, '..', 'experiment-archive'))
if not os.path.exists(new_home):
make_dir(new_home)
shutil.move(record_dir, new_home)
new_record_path = os.path.join(new_home, record_name)
assert os.path.exists(new_record_path)
return new_record_path
def record_experiment(identifier='%T-%N', name='unnamed', print_to_console=True, show_figs=None,
save_figs=True, saved_figure_ext='.fig.pkl', use_temp_dir=False, date=None, prefix=None):
"""
:param identifier: The string that uniquely identifies this experiment record. Convention is that it should be in
the format
:param name: Base-name of the experiment
:param print_to_console: If True, print statements still go to console - if False, they're just rerouted to file.
:param show_figs: Show figures when the experiment produces them. Can be:
'hang': Show and hang
'draw': Show but keep on going
False: Don't show figures
"""
# Note: matplotlib imports are internal in order to avoid trouble for people who may import this module without having
# a working matplotlib (which can occasionally be tricky to install).
if date is None:
date = datetime.now()
identifier = format_filename(file_string=identifier, base_name=name, current_time=date)
if show_figs is None:
show_figs = 'draw' if is_test_mode() else 'hang'
assert show_figs in ('hang', 'draw', False)
if use_temp_dir:
experiment_directory = tempfile.mkdtemp()
atexit.register(lambda: shutil.rmtree(experiment_directory))
else:
experiment_directory = get_local_experiment_path(identifier)
make_dir(experiment_directory)
this_record = ExperimentRecord(experiment_directory)
# Create context that sets the current experiment record
# and the context which captures stdout (print statements) and logs them.
contexts = [
hold_current_experiment_record(this_record),
CaptureStdOut(log_file_path=os.path.join(experiment_directory, 'output.txt'), print_to_console=print_to_console, prefix=prefix)
]
if is_matplotlib_imported():
from artemis.plotting.manage_plotting import WhatToDoOnShow
# Add context that modifies how matplotlib figures are shown.
contexts.append(WhatToDoOnShow(show_figs))
if save_figs:
from artemis.plotting.saving_plots import SaveFiguresOnShow
# Add context that saves figures when show is called.
contexts.append(SaveFiguresOnShow(path=os.path.join(experiment_directory, 'fig-%T-%L' + saved_figure_ext)))
with nested(*contexts):
yield this_record
def archive_record(record):
"""
:param ExperimentRecord record:
:return str: New directory
"""
record_dir = record.get_dir()
exp_dir, record_name = os.path.split(record_dir)
new_home = os.path.normpath(
os.path.join(exp_dir, '..', 'experiment-archive'))
if not os.path.exists(new_home):
make_dir(new_home)
shutil.move(record_dir, new_home)
new_record_path = os.path.join(new_home, record_name)
assert os.path.exists(new_record_path)
return new_record_path
def demo_optimize_conv_scales(n_epochs=5,
comp_weight=1e-11,
learning_rate=0.1,
error_loss='KL',
use_softmax=True,
optimizer='sgd',
shuffle_training=False):
"""
Run the scale optimization routine on a convnet.
:param n_epochs:
:param comp_weight:
:param learning_rate:
:param error_loss:
:param use_softmax:
:param optimizer:
:param shuffle_training:
:return:
"""
if error_loss == 'KL' and not use_softmax:
raise Exception(
"It's very strange that you want to use a KL divergence on something other than a softmax error. I assume you've made a mistake."
)
training_videos, training_vgg_inputs = get_vgg_video_splice(
['ILSVRC2015_train_00033010', 'ILSVRC2015_train_00336001'],
shuffle=shuffle_training,
shuffling_rng=1234)
test_videos, test_vgg_inputs = get_vgg_video_splice(
['ILSVRC2015_train_00033009', 'ILSVRC2015_train_00033007'])
set_dbplot_figure_size(12, 6)
n_frames_to_show = 10
display_frames = np.arange(
len(test_videos) / n_frames_to_show / 2, len(test_videos),
len(test_videos) / n_frames_to_show)
ax1 = dbplot(np.concatenate(test_videos[display_frames], axis=1),
"Test Videos",
title='',
plot_type='pic')
plt.subplots_adjust(wspace=0, hspace=.05)
ax1.set_xticks(224 * np.arange(len(display_frames) / 2) * 2 + 224 / 2)
ax1.tick_params(labelbottom='on')
layers = get_vgg_layer_specifiers(
up_to_layer='prob' if use_softmax else 'fc8')
# Setup the true VGGnet and get the outputs
f_true = ConvNet.from_init(layers, input_shape=(3, 224, 224)).compile()
true_test_out = flatten2(
np.concatenate([
f_true(frame_positions[None])
for frame_positions in test_vgg_inputs
]))
top5_true_guesses = argtopk(true_test_out, 5)
true_guesses = np.argmax(true_test_out, axis=1)
true_labels = [
get_vgg_label_at(g, short=True)
for g in true_guesses[display_frames[::2]]
]
full_convnet_cost = np.array([
get_full_convnet_computational_cost(layer_specs=layers,
input_shape=(3, 224, 224))
] * len(test_videos))
# Setup the approximate networks
slrc_net = ScaleLearningRoundingConvnet.from_convnet_specs(
layers,
optimizer=get_named_optimizer(optimizer, learning_rate=learning_rate),
corruption_type='rand',
rng=1234)
f_train_slrc = slrc_net.train_scales.partial(
comp_weight=comp_weight, error_loss=error_loss).compile()
f_get_scales = slrc_net.get_scales.compile()
round_fp = RoundConvNetForwardPass(layers)
sigmadelta_fp = SigmaDeltaConvNetForwardPass(layers,
input_shape=(3, 224, 224))
p = ProgressIndicator(n_epochs * len(training_videos))
output_dir = make_dir(get_local_path('output/%T-convnet-spikes'))
for input_minibatch, minibatch_info in minibatch_iterate_info(
training_vgg_inputs,
n_epochs=n_epochs,
minibatch_size=1,
test_epochs=np.arange(0, n_epochs, 0.1)):
if minibatch_info.test_now:
with EZProfiler('test'):
current_scales = f_get_scales()
round_cost, round_out = round_fp.get_cost_and_output(
test_vgg_inputs, scales=current_scales)
sd_cost, sd_out = sigmadelta_fp.get_cost_and_output(
test_vgg_inputs, scales=current_scales)
round_guesses, round_top1_correct, round_top5_correct = get_and_report_scores(
round_cost,
round_out,
name='Round',
true_top_1=true_guesses,
true_top_k=top5_true_guesses)
sd_guesses, sd_top1_correct, sd_top5_correct = get_and_report_scores(
sd_cost,
sd_out,
name='SigmaDelta',
true_top_1=true_guesses,
true_top_k=top5_true_guesses)
round_labels = [
get_vgg_label_at(g, short=True)
for g in round_guesses[display_frames[::2]]
]
ax1.set_xticklabels([
'{}\n{}'.format(tg, rg)
for tg, rg in izip_equal(true_labels, round_labels)
])
ax = dbplot(
np.array([
round_cost / 1e9, sd_cost / 1e9,
full_convnet_cost / 1e9
]).T,
'Computation',
plot_type='thick-line',
ylabel='GOps',
title='',
legend=['Round', '$\Sigma\Delta$', 'Original'],
)
ax.set_xticklabels([])
plt.grid()
dbplot(
100 * np.array(
[cummean(sd_top1_correct),
cummean(sd_top5_correct)]).T,
"Score",
plot_type=lambda: LinePlot(
y_bounds=(0, 100),
plot_kwargs=[
dict(linewidth=3, color='k'),
dict(linewidth=3, color='k', linestyle=':')
]),
title='',
legend=[
'Round/$\Sigma\Delta$ Top-1',
'Round/$\Sigma\Delta$ Top-5'
],
ylabel='Cumulative\nPercent Accuracy',
xlabel='Frame #',
layout='v',
)
plt.grid()
plt.savefig(
os.path.join(output_dir,
'epoch-%.3g.pdf' % (minibatch_info.epoch, )))
f_train_slrc(input_minibatch)
p()
print "Epoch {:3.2f}: Scales: {}".format(
minibatch_info.epoch, ['%.3g' % float(s) for s in f_get_scales()])
results = dict(current_scales=current_scales,
round_cost=round_cost,
round_out=round_out,
sd_cost=sd_cost,
sd_out=sd_out,
round_guesses=round_guesses,
round_top1_correct=round_top1_correct,
round_top5_correct=round_top5_correct,
sd_guesses=sd_guesses,
sd_top1_correct=sd_top1_correct,
sd_top5_correct=sd_top5_correct)
dbplot_hang()
return results
def get_experiment_dir():
path = os.path.expanduser(get_artemis_config_value(section="experiments", option="experiment_directory", write_default=True, default_generator=lambda: get_artemis_data_path('experiments')))
if not os.path.exists(path):
make_dir(path)
return path