def plots(self, models, keys):
"""
Plots.
"""
labels = [
'Normal', 'OffAdvTrain', 'OnAdvTrain', 'STNAugm', 'STNAdvTrain',
'OnClassAdvTrain'
]
if self.args.learned_on_data:
labels += ['OnDataAdvTrain']
norms = [1, 2, float('inf')]
attacks = keys
for model in models:
log('[Experiment] %s' % model)
for attack in attacks:
n = 2
errors = self.results[model][:, :, 1].flatten()
distances = self.results['%s_%s' %
(model, attack)][:, :, 0, n,
2].flatten()
successes = self.results['%s_%s' %
(model, attack)][:, :, 0, 0,
0].flatten()
distance_slope, _, distance_correlation, distance_p_value, _ = scipy.stats.linregress(
errors, distances)
success_slope, _, success_correlation, success_p_value, _ = scipy.stats.linregress(
errors, successes)
log('[Experiment] %s: distance=%g (corr=%g, p=%g) success=%g (corr=%g, p=%g)'
% (attack, distance_slope, distance_correlation,
distance_p_value, success_slope, success_correlation,
success_p_value))
# Standard error
x = numpy.stack([self.statistics[model][:, 0] for model in models],
axis=1).T
y = numpy.stack([self.statistics[model][:, 1] for model in models],
axis=1).T
plot_file = paths.image_file('0_evaluation/plot_error')
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file('0_evaluation/latex_error')
latex.line(latex_file, x, y, labels)
for attack in attacks:
# Success rate
x = numpy.stack([self.statistics[model][:, 0] for model in models],
axis=1).T
y = numpy.stack([
self.statistics['%s_%s' % (model, attack)][:, 0, 0, 0]
for model in models
],
axis=1).T
plot_file = paths.image_file('0_evaluation/plot_%s_success_rate' %
attack)
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file(
'0_evaluation/latex_%s_success_rate' % attack)
latex.line(latex_file, x, y, labels)
# L_inf Distances Off-Manifold
n = 2
x = numpy.stack([self.statistics[model][:, 0] for model in models],
axis=1).T
y = numpy.stack([
self.statistics['%s_%s' % (model, attack)][:, 0, n, 2]
for model in models
],
axis=1).T
plot_file = paths.image_file('0_evaluation/plot_%s_distance_%g' %
(attack, norms[n]))
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file('0_evaluation/latex_%s_distance_%g' %
(attack, norms[n]))
latex.line(latex_file, x, y, labels)
# Error and success rate
c = numpy.stack(
[self.results[model][:, :, 0].flatten() for model in models],
axis=1).T
x = numpy.stack(
[self.results[model][:, :, 1].flatten() for model in models],
axis=1).T
y = numpy.stack([
self.results['%s_%s' %
(model, attack)][:, :, 0, 0, 0].flatten()
for model in models
],
axis=1).T
plot_file = paths.image_file(
'0_evaluation/plot_%s_error_success_rate' % attack)
plot.scatter2(plot_file, x, y, labels)
latex_file = paths.latex_file(
'0_evaluation/latex_%s_error_success_rate' % attack)
latex.scatter2(latex_file, x, y, labels, c)
# Error and success rate as line
c = numpy.stack([self.statistics[model][:, 0] for model in models],
axis=1).T
x = numpy.stack([self.statistics[model][:, 1] for model in models],
axis=1).T
y = numpy.stack([
self.statistics['%s_%s' % (model, attack)][:, 0, 0, 0]
for model in models
],
axis=1).T
plot_file = paths.image_file(
'0_evaluation/plot_%s_error_success_rate_line' % attack)
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file(
'0_evaluation/latex_%s_error_success_rate_line' % attack)
latex.line(latex_file, x, y, labels, c)
def plots(self, models, keys):
"""
Plots.
"""
labels = [
'Normal', 'OffAdvTrain', 'OnAdvTrain', 'STNAugm', 'STNAdvTrain',
'OnClassAdvTrain'
]
norms = [1, 2, float('inf')]
attack_types = ['madry', 'cw']
attacks = []
attacks.append('off')
attacks.append('on')
attacks.append('learned_on_class')
# Standard error
x = numpy.stack([self.statistics[model][:, 0] for model in models],
axis=1).T
y = numpy.stack([self.statistics[model][:, 1] for model in models],
axis=1).T
plot_file = paths.image_file('0_evaluation/plot_error')
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file('0_evaluation/latex_error')
latex.line(latex_file, x, y, labels)
for a in range(len(attack_types)):
for attack in attacks:
# Success rate
x = numpy.stack(
[self.statistics[model][:, 0] for model in models],
axis=1).T
y = numpy.stack([
self.statistics['%s_%s' % (model, attack)][:, a, 0, 0]
for model in models
],
axis=1).T
plot_file = paths.image_file(
'0_evaluation/plot_%s_%s_success_rate' %
(attack_types[a], attack))
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file(
'0_evaluation/latex_%s_%s_success_rate' %
(attack_types[a], attack))
latex.line(latex_file, x, y, labels)
# L_inf Distances Off-Manifold
n = 2
x = numpy.stack(
[self.statistics[model][:, 0] for model in models],
axis=1).T
y = numpy.stack([
self.statistics['%s_%s' % (model, attack)][:, a, n, 2]
for model in models
],
axis=1).T
plot_file = paths.image_file(
'0_evaluation/plot_%s_%s_distance_%g' %
(attack_types[a], attack, norms[n]))
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file(
'0_evaluation/latex_%s_%s_distance_%g' %
(attack_types[a], attack, norms[n]))
latex.line(latex_file, x, y, labels)
# Error and success rate
c = numpy.stack([
self.results[model][:, :, 0].flatten() for model in models
],
axis=1).T
x = numpy.stack([
self.results[model][:, :, 1].flatten() for model in models
],
axis=1).T
y = numpy.stack([
self.results['%s_%s' %
(model, attack)][:, :, a, 0, 0].flatten()
for model in models
],
axis=1).T
plot_file = paths.image_file(
'0_evaluation/plot_%s_%s_error_success_rate' %
(attack_types[a], attack))
plot.scatter2(plot_file, x, y, labels)
latex_file = paths.latex_file(
'0_evaluation/latex_%s_%s_error_success_rate' %
(attack_types[a], attack))
latex.scatter2(latex_file, x, y, labels, c)
# Error and success rate as line
c = numpy.stack(
[self.statistics[model][:, 0] for model in models],
axis=1).T
x = numpy.stack(
[self.statistics[model][:, 1] for model in models],
axis=1).T
y = numpy.stack([
self.statistics['%s_%s' % (model, attack)][:, a, 0, 0]
for model in models
],
axis=1).T
plot_file = paths.image_file(
'0_evaluation/plot_%s_%s_error_success_rate_line' %
(attack_types[a], attack))
plot.line(plot_file, x, y, labels)
latex_file = paths.latex_file(
'0_evaluation/latex_%s_%s_error_success_rate_line' %
(attack_types[a], attack))
latex.line(latex_file, x, y, labels, c)