def exec(command_args, prog, description):
delimiter = "--"
usage = f"armory exec <docker image> [-d] [--use-gpu] {delimiter} <exec command>"
parser = argparse.ArgumentParser(prog=prog,
description=description,
usage=usage)
_docker_image(parser)
_debug(parser)
_use_gpu(parser)
_gpus(parser)
try:
index = command_args.index(delimiter)
except ValueError:
print(f"ERROR: delimiter '{delimiter}' is required.")
parser.print_help()
sys.exit(1)
exec_args = command_args[index + 1:]
armory_args = command_args[:index]
if exec_args:
command = " ".join(exec_args)
else:
print("ERROR: exec command required")
parser.print_help()
sys.exit(1)
args = parser.parse_args(armory_args)
coloredlogs.install(level=args.log_level)
config = {"sysconfig": {"docker_image": args.docker_image}}
_set_gpus(config, args.use_gpu, args.gpus)
rig = Evaluator(config)
rig.run(command=command)
def download(command_args, prog, description):
"""
Script to download all datasets and model weights for offline usage.
"""
parser = argparse.ArgumentParser(prog=prog, description=description)
_debug(parser)
parser.add_argument(
metavar="<download data config file>",
dest="download_config",
type=str,
action=DownloadConfig,
help=
f"Configuration for download of data. See {DEFAULT_SCENARIO}. Note: file must be under current working directory.",
)
parser.add_argument(
metavar="<scenario>",
dest="scenario",
type=str,
default="all",
help=
"scenario for which to download data, 'list' for available scenarios, or blank to download all scenarios",
nargs="?",
)
_no_docker(parser)
args = parser.parse_args(command_args)
coloredlogs.install(level=args.log_level)
if args.no_docker:
logger.info(
"Downloading requested datasets and model weights in host mode...")
paths.set_mode("host")
from armory.data import datasets
from armory.data import model_weights
datasets.download_all(args.download_config, args.scenario)
model_weights.download_all(args.download_config, args.scenario)
return
if not armory.is_dev():
logger.info("Downloading all docker images....")
_pull_docker_images()
logger.info("Downloading requested datasets and model weights...")
config = {"sysconfig": {"docker_image": images.TF1}}
rig = Evaluator(config)
cmd = "; ".join([
"import logging",
"import coloredlogs",
f"coloredlogs.install({args.log_level})",
"from armory.data import datasets",
"from armory.data import model_weights",
f'datasets.download_all("{args.download_config}", "{args.scenario}")',
f'model_weights.download_all("{args.download_config}", "{args.scenario}")',
])
rig.run(command=f"python -c '{cmd}'")
def exec(command_args, prog, description):
delimiter = "--"
usage = "armory exec <docker image> [-d] [--use-gpu] -- <exec command>"
parser = argparse.ArgumentParser(prog=prog,
description=description,
usage=usage)
parser.add_argument(
"docker_image",
metavar="<docker image>",
type=str,
help="docker image framework: 'tf1', 'tf2', or 'pytorch'",
action=DockerImage,
)
parser.add_argument(
"-d",
"--debug",
dest="log_level",
action="store_const",
const=logging.DEBUG,
default=logging.INFO,
help="Debug output (logging=DEBUG)",
)
parser.add_argument(
"--use-gpu",
dest="use_gpu",
action="store_const",
const=True,
default=False,
help="Whether to use GPU when launching",
)
try:
index = command_args.index(delimiter)
except ValueError:
print(f"ERROR: delimiter '{delimiter}' is required.")
parser.print_help()
sys.exit(1)
exec_args = command_args[index + 1:]
armory_args = command_args[:index]
if not exec_args:
print("ERROR: exec command required")
parser.print_help()
sys.exit(1)
args = parser.parse_args(armory_args)
coloredlogs.install(level=args.log_level)
paths.host()
config = {
"sysconfig": {
"use_gpu": args.use_gpu,
"docker_image": args.docker_image,
}
}
rig = Evaluator(config)
rig.run(command=" ".join(exec_args))
def run(command_args, prog, description):
parser = argparse.ArgumentParser(prog=prog, description=description)
parser.add_argument("filepath",
metavar="<json_config>",
type=str,
help="json config file")
_debug(parser)
_interactive(parser)
_jupyter(parser)
_port(parser)
_use_gpu(parser)
_gpus(parser)
_no_docker(parser)
parser.add_argument(
"--output-dir",
type=str,
help="Override of default output directory prefix",
)
parser.add_argument(
"--output-filename",
type=str,
help="Override of default output filename prefix",
)
parser.add_argument(
"--check",
action="store_true",
help="Whether to quickly check to see if scenario code runs",
)
args = parser.parse_args(command_args)
coloredlogs.install(level=args.log_level)
try:
config = load_config(args.filepath)
except ValidationError as e:
logger.error(
f"Could not validate config: {e.message} @ {'.'.join(e.absolute_path)}"
)
sys.exit(1)
except json.decoder.JSONDecodeError:
logger.exception(f"Could not decode {args.filepath} as a json file.")
if not args.filepath.lower().endswith(".json"):
logger.warning(f"{args.filepath} is not a '*.json' file")
sys.exit(1)
_set_gpus(config, args.use_gpu, args.gpus)
_set_outputs(config, args.output_dir, args.output_filename)
rig = Evaluator(config, no_docker=args.no_docker)
rig.run(
interactive=args.interactive,
jupyter=args.jupyter,
host_port=args.port,
check_run=args.check,
)
def run(command_args, prog, description):
parser = argparse.ArgumentParser(prog=prog, description=description)
parser.add_argument("filepath",
metavar="<json_config>",
type=str,
help="json config file")
parser.add_argument(
"-d",
"--debug",
dest="log_level",
action="store_const",
const=logging.DEBUG,
default=logging.INFO,
help="Debug output (logging=DEBUG)",
)
parser.add_argument(
"-i",
"--interactive",
dest="interactive",
action="store_const",
const=True,
default=False,
help="Whether to allow interactive access to container",
)
parser.add_argument(
"-j",
"--jupyter",
dest="jupyter",
action="store_const",
const=True,
default=False,
help="Whether to set up Jupyter notebook from container",
)
parser.add_argument(
"-p",
"--port",
dest="port",
type=int,
action=PortNumber,
metavar="",
default=8888,
help="Port number {0, ..., 65535} to connect to Jupyter on",
)
args = parser.parse_args(command_args)
coloredlogs.install(level=args.log_level)
paths.host()
rig = Evaluator(args.filepath)
rig.run(interactive=args.interactive,
jupyter=args.jupyter,
host_port=args.port)
def launch(command_args, prog, description):
parser = argparse.ArgumentParser(prog=prog, description=description)
_docker_image(parser)
_debug(parser)
_interactive(parser)
_jupyter(parser)
_port(parser)
_use_gpu(parser)
_no_gpu(parser)
_gpus(parser)
_root(parser)
args = parser.parse_args(command_args)
coloredlogs.install(level=args.log_level)
config = {"sysconfig": {"docker_image": args.docker_image}}
_set_gpus(config, args.use_gpu, args.no_gpu, args.gpus)
rig = Evaluator(config, root=args.root)
exit_code = rig.run(
interactive=args.interactive,
jupyter=args.jupyter,
host_port=args.port,
command="true # No-op",
)
sys.exit(exit_code)
def run(command_args, prog, description):
parser = argparse.ArgumentParser(prog=prog, description=description)
parser.add_argument(
"filepath",
metavar="<json_config>",
type=str,
help="json config file. Use '-' to accept standard input or pipe.",
)
_debug(parser)
_interactive(parser)
_jupyter(parser)
_port(parser)
_use_gpu(parser)
_no_gpu(parser)
_gpus(parser)
_no_docker(parser)
_root(parser)
parser.add_argument(
"--output-dir",
type=str,
help="Override of default output directory prefix",
)
parser.add_argument(
"--output-filename",
type=str,
help="Override of default output filename prefix",
)
parser.add_argument(
"--check",
action="store_true",
help="Whether to quickly check to see if scenario code runs",
)
parser.add_argument(
"--num-eval-batches",
type=int,
help=
"Number of batches to use for evaluation of benign and adversarial examples",
)
parser.add_argument(
"--skip-benign",
action="store_true",
help="Skip benign inference and metric calculations",
)
parser.add_argument(
"--skip-attack",
action="store_true",
help="Skip attack generation and metric calculations",
)
parser.add_argument(
"--validate-config",
action="store_true",
help="Validate model configuration against several checks",
)
args = parser.parse_args(command_args)
coloredlogs.install(level=args.log_level)
try:
if args.filepath == "-":
if sys.stdin.isatty():
logging.error(
"Cannot read config from raw 'stdin'; must pipe or redirect a file"
)
sys.exit(1)
logger.info("Reading config from stdin...")
config = load_config_stdin()
else:
config = load_config(args.filepath)
except ValidationError as e:
logger.error(
f"Could not validate config: {e.message} @ {'.'.join(e.absolute_path)}"
)
sys.exit(1)
except json.decoder.JSONDecodeError:
if args.filepath == "-":
logger.error("'stdin' did not provide a json-parsable input")
else:
logger.error(f"Could not decode '{args.filepath}' as a json file.")
if not args.filepath.lower().endswith(".json"):
logger.warning(f"{args.filepath} is not a '*.json' file")
sys.exit(1)
_set_gpus(config, args.use_gpu, args.no_gpu, args.gpus)
_set_outputs(config, args.output_dir, args.output_filename)
rig = Evaluator(config, no_docker=args.no_docker, root=args.root)
exit_code = rig.run(
interactive=args.interactive,
jupyter=args.jupyter,
host_port=args.port,
check_run=args.check,
num_eval_batches=args.num_eval_batches,
skip_benign=args.skip_benign,
skip_attack=args.skip_attack,
validate_config=args.validate_config,
)
sys.exit(exit_code)
def launch(command_args, prog, description):
parser = argparse.ArgumentParser(prog=prog, description=description)
parser.add_argument(
"docker_image",
metavar="<docker image>",
type=str,
help="docker image framework: 'tf1', 'tf2', or 'pytorch'",
action=DockerImage,
)
parser.add_argument(
"-d",
"--debug",
dest="log_level",
action="store_const",
const=logging.DEBUG,
default=logging.INFO,
help="Debug output (logging=DEBUG)",
)
parser.add_argument(
"-i",
"--interactive",
dest="interactive",
action="store_const",
const=True,
default=False,
help="Whether to allow interactive access to container",
)
parser.add_argument(
"-j",
"--jupyter",
dest="jupyter",
action="store_const",
const=True,
default=False,
help="Whether to set up Jupyter notebook from container",
)
parser.add_argument(
"-p",
"--port",
dest="port",
type=int,
action=PortNumber,
metavar="",
default=8888,
help="Port number {0, ..., 65535} to connect to Jupyter on",
)
parser.add_argument(
"--use-gpu",
dest="use_gpu",
action="store_const",
const=True,
default=False,
help="Whether to use GPU when launching",
)
args = parser.parse_args(command_args)
coloredlogs.install(level=args.log_level)
paths.host()
config = {
"sysconfig": {
"use_gpu": args.use_gpu,
"docker_image": args.docker_image,
}
}
rig = Evaluator(config)
rig.run(interactive=args.interactive,
jupyter=args.jupyter,
host_port=args.port)