def _report_attack_results(self):
"""
Create subsubsection describing the most important results of the attack.
This subsection contains results only for the first target model.
"""
tm = 0 # Specify target model
self.report_section.append(Subsubsection("Attack Results"))
res = self.attack_results
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
self.report_section.append(Command("centering"))
with self.report_section.create(Tabular("|l|c|")) as result_tab:
result_tab.add_hline()
result_tab.add_row(["True Positives", round(res["tp"][tm], 3)])
result_tab.add_hline()
result_tab.add_row(["True Negatives", round(res["tn"][tm], 3)])
result_tab.add_hline()
result_tab.add_row(["False Positives", round(res["fp"][tm], 3)])
result_tab.add_hline()
result_tab.add_row(["False Negatives", round(res["fn"][tm], 3)])
result_tab.add_hline()
result_tab.add_row(["Accuracy", round(res["accuracy"][tm], 3)])
result_tab.add_hline()
result_tab.add_row(["Precision", round(res["precision"][tm], 3)])
result_tab.add_hline()
result_tab.add_row(["Recall", round(res["recall"][tm], 3)])
result_tab.add_hline()
self.report_section.append(Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command("captionof", "table", extra_arguments="Attack Summary")
)
def _report_attack_configuration(self):
"""
Create subsubsection about the attack and data configuration.
"""
# Create tables for attack parameters and the data configuration.
tm = 0 # Specify target model
dc = self.data_conf
self.report_section.append(Subsubsection("Attack Details"))
with self.report_section.create(MiniPage()):
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
self.report_section.append(Command("centering"))
temp_pars_desc = self.pars_descriptors.copy()
if "attack_model" in temp_pars_desc:
del temp_pars_desc["attack_model"]
values = self.inference_attacks[tm].__dict__.copy()
if hasattr(self, "hopskipjump_args"):
logger.debug("Include HopSkipJump params")
values.update(self.hopskipjump_args)
report.create_attack_pars_table(
self.report_section,
values,
temp_pars_desc,
)
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
# -- Create table for the data configuration
self.report_section.append(Command("centering"))
nr_targets, attack_data_size = dc["attack_record_indices"].shape
with self.report_section.create(Tabular("|l|c|")) as tab_dc:
tab_dc.add_hline()
tab_dc.add_row(["Attacked target models", nr_targets])
tab_dc.add_hline()
if "train_record_indices" in dc:
_, train_data_size = dc["train_record_indices"].shape
tab_dc.add_row(["Training set size", train_data_size])
tab_dc.add_hline()
if "test_record_indices" in dc:
_, test_data_size = dc["test_record_indices"].shape
tab_dc.add_row(["Test set size", test_data_size])
tab_dc.add_hline()
tab_dc.add_row(["Attack set size", attack_data_size])
tab_dc.add_hline()
self.report_section.append(Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"table",
extra_arguments="Target and Data Configuration",
)
)
def _report_attack_results(self, save_path):
"""
Create subsubsection describing the most important results of the attack.
Parameters
----------
save_path :
Path to save the tex, pdf and asset files of the attack report.
This subsection contains results only for the first target model.
"""
tm = 0 # Specify target model
self.report_section.append(Subsubsection("Attack Results"))
res = self.attack_results
# Histogram
path = report.plot_class_dist_histogram(
self.attack_alias, res["ec_accuracy_list"][tm], save_path
)
with self.report_section.create(MiniPage()):
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
self.report_section.append(Command("centering"))
self.report_section.append(
Command(
"includegraphics",
NoEscape(path),
"width=8cm",
)
)
self.report_section.append(Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"figure",
extra_arguments="Evaluation Accuracy Distribution",
)
)
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
self.report_section.append(Command("centering"))
with self.report_section.create(Tabular("|l|c|")) as result_tab:
result_tab.add_hline()
result_tab.add_row(
["Extracted model accuracy", round(res["ec_accuracy"][tm], 3)]
)
result_tab.add_hline()
self.report_section.append(Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command("captionof", "table", extra_arguments="Attack Summary")
)
def _report_attack_results(self, save_path):
"""
Create subsubsection describing the most important results of the attack.
Parameters
----------
save_path :
Path to save the tex, pdf and asset files of the attack report.
This subsection contains results only for the first target model.
"""
tm = 0 # Specify target model
self.report_section.append(Subsubsection("Attack Results"))
res = self.attack_results
# Plot image grid
nb_smaples = min(10, len(res["inferred_training_samples"][tm]))
ncols = min(10, nb_smaples)
fig, axes = plt.subplots(nrows=1, ncols=ncols, figsize=(15, 5))
for i in range(nb_smaples):
image = res["inferred_training_samples"][tm][i]
ax = axes[i]
ax.set_xticks([])
ax.set_yticks([])
ax.set_xlabel(f"Example {i}")
ax.imshow(image)
alias_no_spaces = str.replace(self.attack_alias, " ", "_")
fig.savefig(
save_path + f"/fig/{alias_no_spaces}-examples.pdf", bbox_inches="tight"
)
plt.close(fig)
with self.report_section.create(Figure(position="H")) as fig:
fig.add_image(
f"fig/{alias_no_spaces}-examples.pdf", width=NoEscape(r"\textwidth")
)
self.report_section.append(Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"figure",
extra_arguments="Inferred Samples (up to 10 samples, not sorted, "
"not selected by any criteria)",
)
)
def _report_attack_configuration(self):
"""
Create subsubsection about the attack and data configuration.
"""
# Create tables for attack parameters and the data configuration.
tm = 0 # Specify target model
dc = self.data_conf
self.report_section.append(Subsubsection("Attack Details"))
with self.report_section.create(MiniPage()):
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
self.report_section.append(Command("centering"))
temp_pars_desc = self.pars_descriptors.copy()
if "verbose" in temp_pars_desc:
del temp_pars_desc["verbose"]
values = self.inference_attacks[tm].__dict__.copy()
report.create_attack_pars_table(
self.report_section,
values,
temp_pars_desc,
)
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
# -- Create table for the data configuration
self.report_section.append(Command("centering"))
with self.report_section.create(Tabular("|l|c|")) as tab_dc:
tab_dc.add_hline()
tab_dc.add_row(["Attacked target models", len(self.target_models)])
tab_dc.add_hline()
if "initial_input_data" in dc:
input_size = dc["initial_input_data"][tm].shape[0]
tab_dc.add_row(["Input data size", input_size])
tab_dc.add_hline()
if "initial_input_targets" in dc:
target_size = len(dc["initial_input_targets"][tm])
tab_dc.add_row(["Target size", target_size])
tab_dc.add_hline()
self.report_section.append(Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"table",
extra_arguments="Target and Data Configuration",
)
)
def _report_attack_results(self, save_path):
"""
Create subsubsection describing the most important results of the attack.
Parameters
----------
save_path :
Path to save the tex, pdf and asset files of the attack report.
This subsection contains results only for the first target model.
"""
tm = 0 # Specify target model
self.report_section.append(Subsubsection("Attack Results"))
res = self.attack_results
# ECDF graph (like in paper)
precision_sorted = np.sort(res["precision_list"], axis=0)[:, tm]
recall_sorted = np.sort(res["recall_list"], axis=0)[:, tm]
py = np.arange(1, len(precision_sorted) + 1) / len(precision_sorted)
ry = np.arange(1, len(recall_sorted) + 1) / len(recall_sorted)
fig = plt.figure()
ax = plt.axes()
ax.set_xlabel("Accuracy")
ax.set_ylabel("Cumulative Fraction of Classes")
ax.plot(precision_sorted, py, "k-", label="Precision")
ax.plot(recall_sorted, ry, "k--", label="Recall")
ax.legend()
alias_no_spaces = str.replace(self.attack_alias, " ", "_")
fig.savefig(save_path + f"fig/{alias_no_spaces}-ecdf.pdf",
bbox_inches="tight")
plt.close(fig)
with self.report_section.create(MiniPage()):
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
self.report_section.append(Command("centering"))
self.report_section.append(
Command(
"includegraphics",
NoEscape(f"fig/{alias_no_spaces}-ecdf.pdf"),
"width=8cm",
))
self.report_section.append(
Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"figure",
extra_arguments="Empirical CDF",
))
tp_row = []
fp_row = []
tn_row = []
fn_row = []
class_row = []
precision_row = []
accuracy_row = []
recall_row = []
# Average column
class_row.append(f"0-{self.attack_pars['number_classes']-1}")
tp_row.append(
np.round(
np.sum(res["tp_list"], axis=0)[tm] / len(res["tp_list"]),
2))
fp_row.append(
np.round(
np.sum(res["fp_list"], axis=0)[tm] / len(res["fp_list"]),
2))
tn_row.append(
np.round(
np.sum(res["tn_list"], axis=0)[tm] / len(res["tn_list"]),
2))
fn_row.append(
np.round(
np.sum(res["fn_list"], axis=0)[tm] / len(res["fn_list"]),
2))
accuracy_row.append(
np.round(
np.sum(res["eval_accuracy_list"], axis=0)[tm] /
len(res["eval_accuracy_list"]),
3,
))
precision_row.append(
np.round(
np.sum(res["precision_list"], axis=0)[tm] /
len(res["precision_list"]),
3,
))
recall_row.append(
np.round(
np.sum(res["recall_list"], axis=0)[tm] /
len(res["recall_list"]), 3))
# Maximum accuracy class
max_class = np.argmax(res["eval_accuracy_list"], axis=0)[tm]
class_row.append(max_class)
tp_row.append(res["tp_list"][max_class][tm])
fp_row.append(res["fp_list"][max_class][tm])
tn_row.append(res["tn_list"][max_class][tm])
fn_row.append(res["fn_list"][max_class][tm])
accuracy_row.append(
np.round(res["eval_accuracy_list"][max_class][tm], 3))
precision_row.append(
np.round(res["precision_list"][max_class][tm], 3))
recall_row.append(np.round(res["recall_list"][max_class][tm], 3))
# Minimum accuracy class
min_class = np.argmin(res["eval_accuracy_list"], axis=0)[tm]
class_row.append(min_class)
tp_row.append(res["tp_list"][min_class][tm])
fp_row.append(res["fp_list"][min_class][tm])
tn_row.append(res["tn_list"][min_class][tm])
fn_row.append(res["fn_list"][min_class][tm])
accuracy_row.append(
np.round(res["eval_accuracy_list"][min_class][tm], 3))
precision_row.append(
np.round(res["precision_list"][min_class][tm], 3))
recall_row.append(np.round(res["recall_list"][min_class][tm], 3))
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
self.report_section.append(Command("centering"))
with self.report_section.create(
Tabular("|l|c|c|c|")) as result_tab:
result_tab.add_hline()
result_tab.add_row(
list(
map(
bold,
[
"",
"Average",
"max. Acc.",
"min. Acc.",
],
)))
result_tab.add_hline()
result_tab.add_row(
["Class", class_row[0], class_row[1], class_row[2]])
result_tab.add_hline()
result_tab.add_row(
["True Positives", tp_row[0], tp_row[1], tp_row[2]])
result_tab.add_hline()
result_tab.add_row(
["False Positives", fp_row[0], fp_row[1], fp_row[2]])
result_tab.add_hline()
result_tab.add_row(
["True Negatives", tn_row[0], tn_row[1], tn_row[2]])
result_tab.add_hline()
result_tab.add_row(
["False Negatives", fn_row[0], fn_row[1], fn_row[2]])
result_tab.add_hline()
result_tab.add_row([
"Accuracy", accuracy_row[0], accuracy_row[1],
accuracy_row[2]
])
result_tab.add_hline()
result_tab.add_row([
"Precision",
precision_row[0],
precision_row[1],
precision_row[2],
])
result_tab.add_hline()
result_tab.add_row([
"Recall", recall_row[0], recall_row[1], recall_row[2]
])
result_tab.add_hline()
self.report_section.append(
Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command("captionof",
"table",
extra_arguments="Attack Summary"))
ap = self.attack_pars
# Histograms
fig, (ax0, ax1, ax2) = plt.subplots(1, 3, sharey=True, figsize=(12, 3))
ax0.hist(res["eval_accuracy_list"][:, tm], edgecolor="black")
ax1.hist(res["precision_list"][:, tm], edgecolor="black")
ax2.hist(res["recall_list"][:, tm], edgecolor="black")
ax0.set_xlabel("Accuracy")
ax1.set_xlabel("Precision")
ax2.set_xlabel("Recall")
ax0.set_ylabel("Number of Classes")
ax0.tick_params(axis="x", labelrotation=45)
ax1.tick_params(axis="x", labelrotation=45)
ax2.tick_params(axis="x", labelrotation=45)
ax0.set_axisbelow(True)
ax1.set_axisbelow(True)
ax2.set_axisbelow(True)
alias_no_spaces = str.replace(self.attack_alias, " ", "_")
fig.savefig(save_path +
f"/fig/{alias_no_spaces}-accuracy_precision_recall.pdf",
bbox_inches="tight")
plt.close(fig)
with self.report_section.create(Figure(position="H")) as fig:
fig.add_image(
f"fig/{alias_no_spaces}-accuracy_precision_recall.pdf",
width=NoEscape(r"\textwidth"))
self.report_section.append(
Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"figure",
extra_arguments=
"Accuracy, precision and recall histogram for all "
"attack models",
))
def _report_attack_configuration(self):
"""Create subsubsection about the attack and data configuration."""
# Create tables for attack parameters and the data configuration.
ap = self.attack_pars
dc = self.data_conf
self.report_section.append(Subsubsection("Attack Details"))
with self.report_section.create(MiniPage()):
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
# -- Create table for the attack parameters.
self.report_section.append(Command("centering"))
with self.report_section.create(Tabular("|l|c|")) as tab_ap:
tab_ap.add_hline()
tab_ap.add_row(["Number of classes", ap["number_classes"]])
tab_ap.add_hline()
tab_ap.add_row([
"Number of shadow models", ap["number_shadow_models"]
])
tab_ap.add_hline()
tab_ap.add_row([
"Shadow training set size",
ap["shadow_training_set_size"],
])
tab_ap.add_hline()
tab_ap.add_row(["Shadow epochs", ap["shadow_epochs"]])
tab_ap.add_hline()
tab_ap.add_row(
["Shadow batch size", ap["shadow_batch_size"]])
tab_ap.add_hline()
tab_ap.add_row(["Attack epochs", ap["attack_epochs"]])
tab_ap.add_hline()
tab_ap.add_row(
["Attack batch size", ap["attack_batch_size"]])
tab_ap.add_hline()
self.report_section.append(
Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"table",
extra_arguments="Attack parameters",
))
with self.report_section.create(MiniPage(width=r"0.49\textwidth")):
# -- Create table for the data configuration
self.report_section.append(Command("centering"))
nr_targets, target_training_set_size = dc[
"record_indices_per_target"].shape
with self.report_section.create(Tabular("|l|c|")) as tab_dc:
tab_dc.add_hline()
tab_dc.add_row([
NoEscape("Samples used to train shadow models ($S$)"),
len(dc["shadow_indices"]),
])
tab_dc.add_hline()
tab_dc.add_row([
NoEscape("Samples used to train target models ($T$)"),
len(dc["target_indices"]),
])
tab_dc.add_hline()
tab_dc.add_row([
"Samples used to evaluate the attack",
len(dc["evaluation_indices"]),
])
tab_dc.add_hline()
tab_dc.add_row(["Attacked target models", nr_targets])
tab_dc.add_hline()
tab_dc.add_row([
"Target model's training sets size",
target_training_set_size
])
tab_dc.add_hline()
tab_dc.add_row([
NoEscape("Size of $S \cap T$"),
len(
set(dc["shadow_indices"])
& set(dc["target_indices"])),
])
tab_dc.add_hline()
self.report_section.append(
Command("captionsetup", "labelformat=empty"))
self.report_section.append(
Command(
"captionof",
"table",
extra_arguments="Target and Data Configuration",
))