def filter_valid_attacks(attacks):
valid_attacks = [
attack_name for attack_name in attacks
if CFState.get_instance().active_target.model_data_type in
CFState.get_instance().loaded_attacks[attack_name].tags
]
return valid_attacks
def show_info(self, args):
if args.target:
target = CFState.get_instance().loaded_targets.get(args.target, None)
if not target:
self.pwarning(f"\n [!] Invalid target '{args.target}'.\n")
return
else:
self.poutput(show_target_info(target))
if not args.target and not args.attack:
target = CFState.get_instance().active_target
if not target:
self.pwarning("\n [!] No active target.\n")
return
else:
self.poutput(show_target_info(target))
return
if args.attack:
attack = CFState.get_instance().loaded_attacks.get(args.attack, None)
if not attack:
self.pwarning(f"\n [!] Invalid attack '{args.attack}'.\n")
return
else:
self.poutput(show_attack_info(attack))
return
def do_reload(self, args):
"""Reload the active target. Helpful when making changes to a target class."""
if not CFState.get_instance().active_target:
self.poutput("\n[!] Not interacting with a target. \n")
return
else:
CFState.get_instance().reload_target()
def _set_prompt(self):
if not CFState.get_instance().active_target:
self.prompt = "counterfit> "
else:
if not CFState.get_instance().active_target.active_attack:
self.prompt = f"{CFState.get_instance().active_target.model_name}> "
else:
self.prompt = f"{CFState.get_instance().active_target.model_name}>{CFState.get_instance().active_target.active_attack.attack_name}> "
def do_use(self, args):
"""Select an attack to use on the active target.
Use 'interact' to select a target first.
"""
if not CFState.get_instance().active_target:
self.pwarning("\n [!] Not interacting with any targets. Try interacting with a target.\n")
return
CFState.get_instance().add_attack_to_active_target(args.attack, args.parameters)
def get_attacks(input_attack):
# Takes input attack name and returns respective attack class object
if not input_attack:
attacks = list(CFState.get_instance().loaded_attacks.keys())
else:
try:
attack_cls = CFState.get_instance().loaded_attacks[input_attack]
except KeyError:
raise ValueError(f"Attack Not Found: {input_attack}")
if CFState.get_instance().active_target.model_data_type not in attack_cls.tags:
raise ValueError(f"Inappropriate attack for target: {input_attack}")
attacks = [input_attack]
return attacks
def do_back(self, args):
"""
Exits the active attack or target.
"""
if not CFState.get_instance().active_target.active_attack:
self.poutput(
f"\n[+] Exiting {CFState.get_instance().active_target.model_name}\n"
)
CFState.get_instance().active_target = None
else:
self.poutput(
f"\n[+] Exiting {CFState.get_instance().active_target.active_attack.attack_id}\n"
)
CFState.get_instance().active_target.active_attack = None
def scan_attacks_on_active_target(self, args):
if CFState.get_instance().active_target is None:
self.pwarning(
"\n [!] Not interacting with a target. Set the active target with `interact`.\n"
)
return
# read arguments
input_attacks = args.attack
num_iters = args.iterations
has_logging_enabled = args.log
has_verbose_enabled = args.verbose
set_params = args.set
class_summary = args.class_summary
# get attacks
try:
attacks = get_attacks(input_attacks)
except ValueError as e:
self.pwarning(f'\n [!] {e}\n')
return
# prefilter for valid attacks
attacks = filter_valid_attacks(attacks)
if len(attacks) == 0:
self.pwarning(
"\n [!] No matching attacks found for target. Did you load a framework?\n"
)
return
self.poutput(
f'\n[+] Running these attacks {num_iters}x each:\n\t{", ".join(attacks)}\n'
)
perform_attacks_on_target(self, attacks, num_iters, has_logging_enabled,
has_verbose_enabled, set_params, class_summary)
def do_save(self, args):
"""save parameters and results (if available) of current target
Requires
* "interact <target>"
"""
if not CFState.get_instance().active_target:
self.pwarning("\n [!] Not interacting with a target. Set the active target with `interact`.\n")
return
module_path = "/".join(CFState.get_instance().active_target.__module__.split(".")[:-1])
if "results" not in os.listdir(module_path):
os.mkdir(f"{module_path}/results")
filename = f"{module_path}/results/{CFState.get_instance().active_target.model_name}_{CFState.get_instance().active_target.target_id}.json"
with open(filename, "w") as outfile:
json.dump(CFState.get_instance().active_target.dump(), outfile, indent=1)
self.poutput(f"\n[+] Successfully wrote {filename}\n")
def show_results(self, args):
try:
summary = get_run_summary(CFState.get_instance().active_target)
except (KeyError, AttributeError, TypeError):
self.pwarning("\n [!] No results found. First 'run' an attack.\n")
return
self.poutput(get_printable_run_summary(summary))
def list_frameworks():
columns: List[Column] = list()
data_list: List[List[Any]] = list()
columns.append(Column("Framework", width=20))
columns.append(Column("# of Attacks", width=30))
for framework, list_of_attacks in CFState.get_instance(
).loaded_frameworks.items():
data_list.append([framework, len(list_of_attacks)])
st = SimpleTable(columns)
print()
print(st.generate_table(data_list, row_spacing=0))
print()
def show_options(self, args):
if args.attack:
try:
attack = CFState.get_instance().loaded_attacks.get(
args.attack, None)
except AttributeError:
self.pwarning("\n [!] Attack not found.\n")
else:
# default to the active attack
try:
attack = CFState.get_instance().active_target.active_attack
except AttributeError:
self.pwarning(
"\n [!] Interact with a target and set an active target first.\n"
)
return
if attack is None:
self.pwarning("\n [!] No active attack. Try 'use'.\n")
return
self.poutput("\n" + show_attack_options(attack))
def list_attacks():
columns: List[Column] = list()
data_list: List[List[Any]] = list()
columns.append(Column("Name", width=25))
columns.append(Column("Type", width=15))
columns.append(Column("Category", width=15))
columns.append(Column("Tags", width=15))
columns.append(Column("Framework", width=10))
for _, attack_obj in CFState.get_instance().loaded_attacks.items():
tags = ", ".join(attack_obj.tags)
data_list.append([
attack_obj.attack_name, attack_obj.attack_type,
attack_obj.category, tags, attack_obj.framework
])
st = SimpleTable(columns)
print()
print(st.generate_table(data_list, row_spacing=0))
print()
def show_sample(self, args):
try:
target = CFState.get_instance().active_target
attack = target.active_attack
except AttributeError:
self.pwarning(
"\n [!] Interact with a target and set an active target first.\n")
return
if attack is None:
self.pwarning("\n [!] No active attack. Try 'use'.\n")
return
if args.index is not None and args.result:
self.pwarning("\n [!] Cannot specify both an index and a result.\n")
return
if args.result:
try:
samples = target.active_attack.results["final"]["input"]
sample_index = [target.active_attack.attack_id] * len(samples)
heading1 = "attack id"
except (KeyError, AttributeError):
self.p("\n [!] No results found. First 'run' an attack.\n")
return
elif args.index is None:
restore_index = attack.sample_index
target.set_attack_samples(attack.sample_index)
samples = attack.samples
target.set_attack_samples(restore_index)
sample_index = attack.sample_index
heading1 = None
else:
restore_index = attack.sample_index
target.set_attack_samples(args.index)
samples = attack.samples
target.set_attack_samples(restore_index)
sample_index = args.index
heading1 = None
self.poutput(
show_current_sample(target, get_printable_batch(target, samples),
heading1, sample_index))
def list_targets():
columns: List[Column] = list()
data_list: List[List[Any]] = list()
columns.append(Column("Name", width=15))
columns.append(Column("Type", width=15))
columns.append(Column("Input Shape", width=15))
columns.append(Column("Location", width=85))
for _, target_obj in CFState.get_instance().loaded_targets.items():
shp = str(target_obj.model_input_shape)
data_list.append([
target_obj.model_name, target_obj.model_data_type, shp,
target_obj.model_endpoint
])
st = SimpleTable(columns)
print()
print(st.generate_table(
data_list,
row_spacing=0,
))
print()
def do_list(self, args):
"""List the available targets, frameworks, or attacks."""
if args.type.lower() == "targets":
list_targets()
elif args.type.lower() == "attacks":
if not CFState.get_instance().loaded_attacks:
self.pwarning(
"\n [!] No frameworks have been loaded. Try 'load <framework>'.\n"
)
else:
list_attacks()
elif args.type.lower() == "frameworks":
list_frameworks()
else:
self.pwarning(
"\n [!] Argument not recognized: expecting 'targets', 'attacks', or 'frameworks'.\n"
)
def do_interact(self, args):
"""Sets the active target."""
CFState.get_instance().set_active_target(args.target)
def show_sample(self, args):
target = CFState.get_instance().active_target
if not target:
self.pwarning(
"\n [!] Not interacting with a target. Set the active target with `interact`.\n"
)
return
if sum([args.surprise, args.index is not None, args.result]) > 1:
self.pwarning(
"\n [!] must specify only one of {surprise, index, result}.\n")
return
elif sum([args.surprise, args.index is not None, args.result]) == 0:
# "show sample" with an active target uses the current sample_index as input
try:
args.index = target.active_attack.sample_index
except (ValueError, AttributeError):
self.pwarning(
"\n [!] No active attack. Please provide valid arguments. Try 'show sample -i 2' or 'show sample -s'.\n"
)
return
if isinstance(args.index, int):
sample_index = args.index
target_x_len = len(target.X) - 1
if sample_index > target_x_len:
self.pwarning(
"\n [!] Entered index is out of bounds. Please provide index in the range {0} and {1} including.\n"
.format(0, target_x_len))
return
samples = get_samples(target, sample_index)
heading1 = None
elif args.surprise:
sample_index = random.randint(0, len(target.X) - 1)
samples = get_samples(target, index=sample_index)
heading1 = None
elif args.result:
attack = target.active_attack
if not attack:
self.pwarning(
"\n [!] No active attack. load respective framework and try 'use'.\n"
)
return
if args.index is not None and args.result:
self.pwarning(
"\n [!] Cannot specify both an index and a result.\n")
return
try:
samples = attack.results["final"]["input"]
sample_index = [attack.attack_id] * len(samples)
heading1 = "attack id"
except (KeyError, AttributeError, TypeError):
self.pwarning("\n [!] No results found. First 'run' an attack.\n")
return
else:
self.pwarning(
"\n [!] Please provide valid arguments. Try 'show sample -i 2' or 'show sample -s'.\n"
)
return
self.poutput(
show_current_sample(target, get_printable_batch(target, samples),
heading1, sample_index))
def do_scan(self, args):
"""scan over all attacks against an active target, using random parameters
Requires
* "interact <target>"
"""
def perform_attacks_on_target(attacks, num_iters, has_logging_enabled, has_verbose_enabled):
# loop over loaded attacks
scans_by_attack = defaultdict(list)
scans_by_label = defaultdict(list)
for attack_name in attacks:
for n in range(num_iters):
if has_verbose_enabled:
self.poutput(f"\n[+] Running {attack_name} iteration {n+1} of {num_iters}")
# set random attack algorithm parameters
CFState.get_instance().add_attack_to_active_target(attack_name, "random")
parameters = CFState.get_instance().active_target.active_attack.parameters
if has_verbose_enabled:
self.poutput(
f"Parameters: {json.dumps(parameters)}"
)
# set random attack sample
CFState.get_instance().active_target.set_attack_samples(
random.randint(0, len(CFState.get_instance().active_target.X) - 1)
)
# Initialize the attack to obtain initail labels
CFState.get_instance().active_target.init_run_attack()
# set random target class depends on targeted or untargeted attack
is_targeted = CFState.get_instance().active_target.active_attack.parameters.get("targeted", False)
if is_targeted:
initial_labels = CFState.get_instance().active_target.active_attack.results['initial']['label']
target_class = random.choice(list(set(CFState.get_instance().active_target.model_output_classes).difference(set(initial_labels))))
target_class_idx = CFState.get_instance().active_target.model_output_classes.index(target_class)
else:
target_class_idx = random.randint(
0, len(CFState.get_instance().active_target.model_output_classes) - 1
)
CFState.get_instance().active_target.active_attack.target_class = target_class_idx
# run the attack
CFState.get_instance().active_target.run_attack(logging=has_logging_enabled)
# update the status and show success
summary = get_run_summary(CFState.get_instance().active_target)
scans_by_attack[attack_name].append(summary)
for lab in summary['initial_label']:
scans_by_label[lab].append(summary)
if has_verbose_enabled:
self.poutput(get_printable_run_summary(summary))
# print scan summary
summary_by_attack = {k : get_scan_summary(v) for k, v in scans_by_attack.items()}
summary_by_label = {k : get_scan_summary(v) for k, v in scans_by_label.items()}
printable = get_printable_scan_summary(summary_by_attack, summary_by_label if args.class_summary else None)
self.poutput(printable)
if CFState.get_instance().active_target is None:
self.pwarning("\n [!] Not interacting with a target. Set the active target with `interact`.\n")
return
input_attack = args.attack
num_iters = args.iterations
has_logging_enabled = args.log
has_verbose_enabled = args.verbose
try:
attacks = [get_attacks(a)[0] for a in input_attack]
except ValueError as e:
self.pwarning(f'\n [!] {e}\n')
return
# prefilter for valid attacks
attacks = filter_valid_attacks(attacks)
if len(attacks) == 0:
self.pwarning("\n [!] No matching attacks found for target. Did you load a framework?\n")
return
self.poutput(f'\n[+] Running these attacks {num_iters}x each:\n\t{", ".join(attacks)}\n')
perform_attacks_on_target(attacks, num_iters, has_logging_enabled, has_verbose_enabled)
from counterfit.core.state import CFState
from counterfit.core.run_scan_utils import get_printable_batch, get_run_summary, get_printable_run_summary
from cmd2.table_creator import Column, SimpleTable, HorizontalAlignment
from typing import Any, List
# show
base_parser = argparse.ArgumentParser()
base_subparsers = base_parser.add_subparsers(
title="subcommands", help="show information about models and targets")
# show info
parser_info = base_subparsers.add_parser(
"info", help="show information about the model and active attack")
parser_info.add_argument(
"--attack",
choices=CFState.get_instance().loaded_attacks.keys(),
default=None,
help="attack to show info (defaults to active attack)",
)
parser_info.add_argument(
"--target",
choices=CFState.get_instance().loaded_targets.keys(),
default=None,
help="target to show info (defaults to active target)",
)
# show options
parser_options = base_subparsers.add_parser(
"options", help="show configuration options for the active attack")
parser_options.add_argument(
"--attack",
def perform_attacks_on_target(self, attacks, num_iters, has_logging_enabled,
has_verbose_enabled, set_params, class_summary):
set_params_tuples = re.findall(r"(\w+)\s?=\s?([\w\.]+)",
" ".join(set_params))
# parse set parameters and put into default dict
set_params_dict = dict(set_params_tuples)
# loop over loaded attacks
scans_by_attack = defaultdict(list)
scans_by_label = defaultdict(list)
for attack_name in attacks:
for n in range(num_iters):
if has_verbose_enabled:
self.poutput(
f"\n[+] Running {attack_name} iteration {n+1} of {num_iters}"
)
# set random attack algorithm parameters
CFState.get_instance().add_attack_to_active_target(
attack_name, "random")
parameters = CFState.get_instance(
).active_target.active_attack.parameters
if has_verbose_enabled:
self.poutput(f"Parameters: {json.dumps(parameters)}")
# set random attack sample only if user hasn't set in the parameters
sample_index = set_params_dict.get('sample_index', None)
if sample_index:
sample_index = int(set_params_dict['sample_index'])
else:
sample_index = random.randint(
0,
len(CFState.get_instance().active_target.X) - 1)
CFState.get_instance().active_target.set_attack_samples(
sample_index)
# Initialize the attack to obtain initail labels
CFState.get_instance().active_target.init_run_attack()
# set random target class depends on targeted or untargeted attack
is_targeted = CFState.get_instance(
).active_target.active_attack.parameters.get("targeted", False)
if is_targeted:
initial_labels = CFState.get_instance(
).active_target.active_attack.results['initial']['label']
target_class = random.choice(
list(
set(CFState.get_instance().active_target.
model_output_classes).difference(
set(initial_labels))))
target_class_idx = CFState.get_instance(
).active_target.model_output_classes.index(target_class)
else:
target_class_idx = random.randint(
0,
len(CFState.get_instance().active_target.
model_output_classes) - 1)
CFState.get_instance(
).active_target.active_attack.target_class = target_class_idx
# run the attack
CFState.get_instance().active_target.run_attack(
logging=has_logging_enabled)
# update the status and show success
summary = get_run_summary(CFState.get_instance().active_target)
scans_by_attack[attack_name].append(summary)
for lab in summary['initial_label']:
scans_by_label[lab].append(summary)
if has_verbose_enabled:
self.poutput(get_printable_run_summary(summary))
# print scan summary
summary_by_attack = {
k: get_scan_summary(v)
for k, v in scans_by_attack.items()
}
summary_by_label = {
k: get_scan_summary(v)
for k, v in scans_by_label.items()
}
printable = get_printable_scan_summary(
summary_by_attack, summary_by_label if class_summary else None)
self.poutput(printable)
def do_set(self, args):
"""Set parameters of the active attack on the active target using param1=val1 param2=val2 notation.
This command replaces built-in "set" command, which is renamed to "setg".
"""
if not CFState.get_instance().active_target:
self.pwarning(
'\n [!] No active target. Try "setg" for setting global arguments.\n'
)
return
if not CFState.get_instance().active_target.active_attack:
self.pwarning('\n [!] No active attack. Try "use <attack>".\n')
return
# 'set' with no options shows current variables, similar to "show options"
if not args.what:
self.pwarning(
'\n [!] No arguments specified. Try "set <param>=<value>".\n')
# create structure to ensure all params are present and ordered properly. Defaults to current params to prevent over writing with default values
params_struct = namedtuple(
"params",
[
i for i in CFState.get_instance().active_target.active_attack.
parameters.keys()
] + ["sample_index", "target_class"],
defaults=list(CFState.get_instance(
).active_target.active_attack.parameters.values()) + [
CFState.get_instance().active_target.active_attack.sample_index,
CFState.get_instance().active_target.active_attack.target_class,
],
)
# parse parameters and new values from the args
try:
params_to_update = re.findall(r"(\w+)\s?=\s?([\w\.]+)",
" ".join(args.what))
except:
self.pwarning("\n [!] Failed to parse arguments.\n")
return
# create default params struct
default_params = {
k: v
for k, v in
CFState.get_instance().active_target.active_attack.default.items()
}
default_params["sample_index"] = CFState.get_instance(
).active_target.active_attack.sample_index
default_params["target_class"] = CFState.get_instance(
).active_target.active_attack.target_class
# ensure all current params exist and are ordered correctly
default_params = params_struct(**default_params)._asdict()
# convert string "True"/"true" and "False"/"false" to boolean
for i, v in enumerate(params_to_update):
if type(default_params.get(v[0], None)) is bool:
if v[1].lower() == "true" or int(v[1]) == 1:
params_to_update[i] = (v[0], True)
elif v[1].lower() == "false" or int(v[1]) == 0:
params_to_update[i] = (v[0], False)
# create new params struct using default values where no new values are spec'd
new_params = params_struct(**{
i[0]: type(default_params.get(i[0], ""))(i[1])
for i in params_to_update
})
# create current params struct
current_params = {
k: v
for k, v in
CFState.get_instance().active_target.active_attack.parameters.items()
}
current_params["sample_index"] = CFState.get_instance(
).active_target.active_attack.sample_index
current_params["target_class"] = CFState.get_instance(
).active_target.active_attack.target_class
# ensure all current params exist and are ordered correctly
current_params = params_struct(**current_params)._asdict()
# separate target_class and sample_index from struct and update the relevant values
CFState.get_instance().active_target.active_attack.parameters = {
k: v
for k, v in zip(new_params._fields[:-2], new_params[:-2])
}
CFState.get_instance(
).active_target.active_attack.sample_index = new_params.sample_index
CFState.get_instance(
).active_target.active_attack.target_class = new_params.target_class
# print info
print_new_params = new_params._asdict()
columns: List[Column] = list()
data_list: List[List[Any]] = list()
columns.append(
Column("Attack Parameter (type)",
width=25,
header_horiz_align=HorizontalAlignment.LEFT,
data_horiz_align=HorizontalAlignment.RIGHT))
columns.append(
Column("Default",
width=12,
header_horiz_align=HorizontalAlignment.CENTER,
data_horiz_align=HorizontalAlignment.LEFT))
columns.append(
Column("Previous",
width=12,
header_horiz_align=HorizontalAlignment.CENTER,
data_horiz_align=HorizontalAlignment.LEFT))
columns.append(
Column("New",
width=12,
header_horiz_align=HorizontalAlignment.CENTER,
data_horiz_align=HorizontalAlignment.LEFT))
for k, default_value in default_params.items():
param = f"{k} ({str(type(default_value).__name__)})"
previous_value = current_params.get(k, "")
new_value = print_new_params.get(k, "")
if new_value != previous_value:
data_list.append([
param,
str(default_value),
str(previous_value),
str(new_value)
])
else:
data_list.append(
[param,
str(default_value)[:11],
str(previous_value)[:11], ""])
st = SimpleTable(columns)
print()
print(st.generate_table(data_list, row_spacing=0))
print()
def do_run(self, args):
"""run an active attack against an active target, using parameters set by "set".
Requires
* "interact <target>"
* "use <attack>"
Default parameters may be override via "set param1=val1 param2=val2"
"""
if CFState.get_instance().active_target is None:
self.pwarning("\n [!] Not interacting with a target. Set the active target with `interact`.\n")
return
if CFState.get_instance().active_target.active_attack is None:
self.pwarning("\n [!] No attack specified. Set the active attack with 'use'.\n")
return
if (
CFState.get_instance().active_target.model_data_type
not in CFState.get_instance().active_target.active_attack.tags
):
self.pwarning(
f"Error: innappropriate attack for target {CFState.get_instance().active_target.model_name}"
)
print()
self.pwarning(
f"{CFState.get_instance().active_target.model_name} is of data type {CFState.get_instance().active_target.model_data_type}"
)
self.pwarning(
f"but, {CFState.get_instance().active_target.active_attack.attack_name} is for models with:"
)
for tag in CFState.get_instance().active_target.active_attack.tags:
self.pwarning(f"\t{tag}")
return
self.poutput(
f"\n[+] Running {CFState.get_instance().active_target.active_attack.attack_name} on {CFState.get_instance().active_target.model_name}\n"
)
CFState.get_instance().active_target.set_attack_samples(
CFState.get_instance().active_target.active_attack.sample_index
)
# Run init_attack followed by run_attack
CFState.get_instance().active_target.init_run_attack()
CFState.get_instance().active_target.run_attack(logging=args.log)
summary = get_run_summary(CFState.get_instance().active_target)
self.poutput(get_printable_run_summary(summary))
import sys
import os
import warnings
from counterfit.core import config
from counterfit.core.state import CFState
from counterfit.core.terminal import Terminal
if sys.version_info < (3, 7):
print("[!] Python 3.7+ is required")
sys.exit(1)
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" # make tensorflow quiet
warnings.filterwarnings("ignore", category=FutureWarning)
if __name__ == "__main__":
terminal = Terminal()
num_attacks = CFState.get_instance().import_frameworks()
num_targets = CFState.get_instance().import_targets()
terminal.load_commands(
) # load commands last. Choices depend on targets and attacks.
print(config.start_banner)
print(f"""
[+] {num_attacks} attacks
[+] {num_targets} targets
""")
sys.exit(terminal.cmdloop())
import cmd2
import argparse
from counterfit.core.state import CFState
parser = argparse.ArgumentParser()
parser.add_argument("attack", choices=CFState.get_instance().loaded_attacks.keys())
parser.add_argument("-p", "--parameters", choices=["default", "random"], default="default")
@cmd2.with_argparser(parser)
@cmd2.with_category("Counterfit Commands")
def do_use(self, args):
"""Select an attack to use on the active target.
Use 'interact' to select a target first.
"""
if not CFState.get_instance().active_target:
self.pwarning("\n [!] Not interacting with any targets. Try interacting with a target.\n")
return
CFState.get_instance().add_attack_to_active_target(args.attack, args.parameters)
import argparse
import cmd2
from counterfit.core.state import CFState
parser = argparse.ArgumentParser()
parser.add_argument("framework",
choices=CFState.get_instance().loaded_frameworks,
default="art")
@cmd2.with_argparser(parser)
@cmd2.with_category("Counterfit Commands")
def do_load(self, args):
"""Loads a framework.
:param framework name: the framework to load.
"""
CFState.get_instance().load_framework(args.framework)
def do_predict(self, args):
"""Predict a single sample for the active target"""
if CFState.get_instance().active_target is None:
self.pwarning("\n [!] must first `interact` with a target.\n")
return
else:
target = CFState.get_instance().active_target
if sum([args.random, args.sample_index is not None, args.result]) > 1:
self.pwarning("\n [!] must specify only one of {random, sample_index, result}.\n")
return
heading1 = "Sample Index"
if args.random:
sample_index = random.randint(0, len(target.X) - 1)
samples = set_attack_samples(target, sample_index)
elif args.sample_index: # default behavior
sample_index = args.sample_index
samples = set_attack_samples(target, sample_index)
elif args.result:
try:
samples = target.active_attack.results['final']['input']
sample_index = [target.active_attack.attack_id] * len(samples)
heading1 = "Attack ID"
except (KeyError, AttributeError):
self.pwarning("\n [!] No results found. First 'run' an attack.\n")
return
elif target.active_attack is not None and target.active_attack.sample_index is not None:
sample_index = target.active_attack.sample_index
samples = set_attack_samples(target, sample_index)
else:
self.pwarning("\n [!] No index sample, setting random index.\n")
sample_index = random.randint(0, len(target.X) - 1)
samples = set_attack_samples(target, sample_index)
result = target._submit(samples)
columns: List[Column] = list()
columns.append(
Column(
heading1,
width=8,
header_horiz_align=HorizontalAlignment.LEFT,
data_horiz_align=HorizontalAlignment.RIGHT,
)
)
columns.append(
Column(
"Sample",
width=60,
header_horiz_align=HorizontalAlignment.CENTER,
data_horiz_align=HorizontalAlignment.RIGHT,
)
)
columns.append(
Column(
"Output Scores\n" + str(target.model_output_classes).replace(',', ''),
width=30,
header_horiz_align=HorizontalAlignment.CENTER,
data_horiz_align=HorizontalAlignment.RIGHT,
)
)
if not hasattr(sample_index, "__iter__"):
sample_index = [sample_index]
samples_str = get_printable_batch(target, samples)
results_str = printable_numpy(result)
data_list: List[List[Any]] = list()
for idx, samp, res in zip(sample_index, samples_str, results_str):
data_list.append([idx, samp, res])
st = SimpleTable(columns)
self.poutput("\n" + st.generate_table(data_list, row_spacing=0) + "\n")
def do_load(self, args):
"""Loads a framework.
:param framework name: the framework to load.
"""
CFState.get_instance().load_framework(args.framework)
def do_new(self, args):
"""Optional wizard to aid in creating a new attack target."""
model_name = questionary.text("Target name:").ask()
model_name = model_name.replace(" ", "")
available_frameworks = list(
CFState.get_instance().loaded_frameworks.keys())
framework_choice = questionary.select("Which framework?",
choices=available_frameworks).ask()
if "textattack" in framework_choice:
framework = "TextTarget"
elif "art" in framework_choice:
framework = "ArtTarget"
else:
raise ValueError("invalid framework")
if framework == "TextTarget":
model_data_type = "text"
elif framework == "ArtTarget":
model_data_type = questionary.select("What data type?",
choices=["numpy", "image"]).ask()
else:
raise ValueError("invalid framework")
if model_name not in os.listdir(config.targets_path):
try:
os.mkdir(f"{config.targets_path}/{model_name}")
open(f"{config.targets_path}/{model_name}/__init__.py",
"w").close()
with open(f"{config.targets_path}/{model_name}/{model_name}.py",
"w") as f:
f.write(f"""
# Generated by counterfit #
from counterfit.core.targets import {framework}
class {model_name.capitalize()}({framework}):
model_name = "{model_name.lower()}"
model_data_type = "{model_data_type}"
model_endpoint = ""
model_input_shape = ()
model_output_classes = []
X = []
def __init__(self):
self.X = []
def __call__(self, x):
return x
""")
CFState.get_instance().import_targets()
except Exception as e:
self.pwarning(f"\n [!] Failed to write target file: {e}.\n")
else:
self.pwarning(
f"\n [!] {model_name} already exists. Choose a new name.\n")
def target_singleton_handler(self):
target_singleton_obj = CFState.get_instance()
return target_singleton_obj
