def _multi_run(runner_class, writer, current_log_folder, args):
"""
Run multiple times an experiment and aggregates the results.
This is particularly useful to counter effects of randomness.
Here multiple runs with same parameters are executed and the results averaged.
Additionally "variance shaded plots" gets to be generated and are visible not only
on FS but also on tensorboard under 'IMAGES'.
Parameters
----------
runner_class : String
This is necessary to know on which class should we run the experiments. Default is runner.image_classification.image_classification
writer: Tensorboard.SummaryWriter
Responsible for writing logs in Tensorboard compatible format.
current_log_folder : String
Path to the output folder. Required for saving the raw data of the plots
generated by the multi-run routine.
args : dict
Contains all command line arguments parsed.
Returns
-------
train_scores : ndarray[float] of size (n, `epochs`)
val_scores : ndarray[float] of size (n, `epochs`+1)
test_score : ndarray[float] of size (n)
Train, Val and Test results for each run (n) and epoch
"""
# Instantiate the scores tables which will stores the results.
train_scores = np.zeros((args.multi_run, args.epochs))
val_scores = np.zeros((args.multi_run, args.epochs + 1))
test_scores = np.zeros(args.multi_run)
# As many times as runs
for i in range(args.multi_run):
logging.info('Multi-Run: {} of {}'.format(i + 1, args.multi_run))
train_scores[i, :], val_scores[
i, :], test_scores[i] = runner_class.single_run(
writer,
run=i,
current_log_folder=current_log_folder,
**args.__dict__)
# Generate and add to tensorboard the shaded plot for train
train_curve = plot_mean_std(arr=train_scores[:i + 1],
suptitle='Multi-Run: Train',
title='Runs: {}'.format(i + 1),
xlabel='Epoch',
ylabel='Score',
ylim=[0, 100.0])
save_image_and_log_to_tensorboard(writer,
tag='train_curve',
image_tensor=train_curve,
global_step=i)
logging.info('Generated mean-variance plot for train')
# Generate and add to tensorboard the shaded plot for va
val_curve = plot_mean_std(x=(np.arange(args.epochs + 1) - 1),
arr=np.roll(val_scores[:i + 1],
axis=1,
shift=1),
suptitle='Multi-Run: Val',
title='Runs: {}'.format(i + 1),
xlabel='Epoch',
ylabel='Score',
ylim=[0, 100.0])
save_image_and_log_to_tensorboard(writer,
tag='val_curve',
image_tensor=val_curve,
global_step=i)
logging.info('Generated mean-variance plot for val')
# Log results on disk
np.save(os.path.join(current_log_folder, 'train_values.npy'),
train_scores)
np.save(os.path.join(current_log_folder, 'val_values.npy'), val_scores)
logging.info('Multi-run values for test-mean:{} test-std: {}'.format(
np.mean(test_scores), np.std(test_scores)))
return train_scores, val_scores, test_scores
def _multi_run(runner_class, writer, current_log_folder, args):
"""
Here multiple runs with same parameters are executed and the results averaged.
Additionally "variance shaded plots" gets to be generated and are visible not only on FS but also on
tensorboard under 'IMAGES'.
Parameters:
-----------
:param runner_class: class
This is necessary to know on which class should we run the experiments. Default is runner.image_classification.image_classification
:param writer: Tensorboard SummaryWriter
Responsible for writing logs in Tensorboard compatible format.
:param args:
Any additional arguments (especially for the runner_class)
:return: float[n, epochs], float[n, epochs], float[n]
Train, Val and Test results for each run (n) and epoch
"""
# Instantiate the scores tables which will stores the results.
train_scores = np.zeros((args.multi_run, args.epochs))
val_scores = np.zeros((args.multi_run, args.epochs + 1))
test_scores = np.zeros(args.multi_run)
# As many times as runs
for i in range(args.multi_run):
logging.info('Multi-Run: {} of {}'.format(i + 1, args.multi_run))
train_scores[i, :], val_scores[
i, :], test_scores[i] = runner_class.single_run(
writer,
run=i,
current_log_folder=current_log_folder,
**args.__dict__)
# Generate and add to tensorboard the shaded plot for train
train_curve = plot_mean_std(arr=train_scores[:i + 1],
suptitle='Multi-Run: Train',
title='Runs: {}'.format(i + 1),
xlabel='Epoch',
ylabel='Score',
ylim=[0, 100.0])
save_image_and_log_to_tensorboard(writer,
tag='train_curve',
image_tensor=train_curve,
global_step=i)
logging.info('Generated mean-variance plot for train')
# Generate and add to tensorboard the shaded plot for va
val_curve = plot_mean_std(x=(np.arange(args.epochs + 1) - 1),
arr=np.roll(val_scores[:i + 1],
axis=1,
shift=1),
suptitle='Multi-Run: Val',
title='Runs: {}'.format(i + 1),
xlabel='Epoch',
ylabel='Score',
ylim=[0, 100.0])
save_image_and_log_to_tensorboard(writer,
tag='val_curve',
image_tensor=val_curve,
global_step=i)
logging.info('Generated mean-variance plot for val')
# Log results on disk
np.save(os.path.join(current_log_folder, 'train_values.npy'),
train_scores)
np.save(os.path.join(current_log_folder, 'val_values.npy'), val_scores)
logging.info('Multi-run values for test-mean:{} test-std: {}'.format(
np.mean(test_scores), np.std(test_scores)))
return train_scores, val_scores, test_scores