def get_madry_model(model_kwargs, wrapper_kwargs, weights_file=None): model = make_madry_model(**model_kwargs) input_ph = model.x_input labels_ph = model.y_input training_ph = tf.placeholder(tf.bool, shape=()) # Restore the checkpoint saver = tf.train.Saver() tf_sess = tf.Session() saved_model_dir = paths.DockerPaths().saved_model_dir filepath = os.path.join(saved_model_dir, weights_file) model_file = tf.train.latest_checkpoint(filepath) saver.restore(tf_sess, model_file) wrapped_model = TFClassifier( input_ph=input_ph, output=model.pre_softmax, labels_ph=labels_ph, loss=model.xent, learning=training_ph, sess=tf_sess, clip_values=(0, 255), **wrapper_kwargs ) return wrapped_model
def _gather_env_variables(self): """ Update the extra env variable dictionary to pass into container or run on host """ self.extra_env_vars["ARMORY_GITHUB_TOKEN"] = os.getenv( "ARMORY_GITHUB_TOKEN", default="") self.extra_env_vars["ARMORY_PRIVATE_S3_ID"] = os.getenv( "ARMORY_PRIVATE_S3_ID", default="") self.extra_env_vars["ARMORY_PRIVATE_S3_KEY"] = os.getenv( "ARMORY_PRIVATE_S3_KEY", default="") self.extra_env_vars["ARMORY_INCLUDE_SUBMISSION_BUCKETS"] = os.getenv( "ARMORY_INCLUDE_SUBMISSION_BUCKETS", default="") if not self.armory_global_config["verify_ssl"]: self.extra_env_vars["VERIFY_SSL"] = "false" if self.config["sysconfig"].get("use_gpu", None): gpus = self.config["sysconfig"].get("gpus") if gpus is not None: self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus if self.config["sysconfig"].get("set_pythonhashseed"): self.extra_env_vars["PYTHONHASHSEED"] = "0" if not self.no_docker: self.extra_env_vars["HOME"] = "/tmp" # Because we may want to allow specification of ARMORY_TORCH_HOME # this constant path is placed here among the other imports if self.no_docker: torch_home = paths.HostPaths().pytorch_dir else: torch_home = paths.DockerPaths().pytorch_dir self.extra_env_vars["TORCH_HOME"] = torch_home self.extra_env_vars[environment.ARMORY_VERSION] = armory.__version__
def __init__( self, image_name, runtime: str = "runc", envs: dict = None, ports: dict = None, command: str = "tail -f /dev/null", user: str = "", ): self.docker_client = docker.from_env(version="auto") host_paths = paths.HostPaths() docker_paths = paths.DockerPaths() container_args = { "runtime": runtime, "remove": True, "detach": True, "volumes": { host_paths.cwd: { "bind": docker_paths.cwd, "mode": "rw" }, host_paths.dataset_dir: { "bind": docker_paths.dataset_dir, "mode": "rw", }, host_paths.local_git_dir: { "bind": docker_paths.local_git_dir, "mode": "rw", }, host_paths.output_dir: { "bind": docker_paths.output_dir, "mode": "rw" }, host_paths.saved_model_dir: { "bind": docker_paths.saved_model_dir, "mode": "rw", }, host_paths.tmp_dir: { "bind": docker_paths.tmp_dir, "mode": "rw" }, }, "shm_size": "16G", } if ports is not None: container_args["ports"] = ports if command is not None: container_args["command"] = command if user: container_args["user"] = user if envs: container_args["environment"] = envs self.docker_container = self.docker_client.containers.run( image_name, **container_args) logger.info( f"ARMORY Instance {self.docker_container.short_id} created.")
def ensure_armory_dirs(request): """ CI doesn't mount volumes """ docker_paths = paths.DockerPaths() saved_model_dir = docker_paths.saved_model_dir dataset_dir = docker_paths.dataset_dir output_dir = docker_paths.output_dir os.makedirs(saved_model_dir, exist_ok=True) os.makedirs(dataset_dir, exist_ok=True) os.makedirs(output_dir, exist_ok=True)
def test_download(): saved_model_dir = paths.DockerPaths().saved_model_dir weights_file = "resnet50_imagenet_v1.h5" filepath = os.path.join(saved_model_dir, weights_file) if os.path.isfile(filepath): os.remove(filepath) download_file_from_s3( "armory-public-data", f"model-weights/{weights_file}", f"{saved_model_dir}/{weights_file}", ) assert os.path.isfile(os.path.join(saved_model_dir, weights_file))
""" Test cases for ARMORY datasets. """ import os import pytest import numpy as np from armory.data import datasets from armory.data import adversarial_datasets from armory import paths DATASET_DIR = paths.DockerPaths().dataset_dir def test__parse_token(): for x in "test", "train[15:20]", "train[:10%]", "train[-80%:]": assert datasets._parse_token(x) == x for x, y in [ ("test[[1, 5, 7]]", "test[1:2]+test[5:6]+test[7:8]"), ("test[[1, 4, 5, 6]]", "test[1:2]+test[4:5]+test[5:6]+test[6:7]"), ]: assert datasets._parse_token(x) == y for x in "", "test[", "test[]", "test[[]]", "[10:11]": with pytest.raises(ValueError): datasets._parse_token(x) with pytest.raises(NotImplementedError):
def __init__( self, image_name, runtime: str = "runc", envs: dict = None, ports: dict = None, command: str = "tail -f /dev/null", ): self.docker_client = docker.from_env(version="auto") host_paths = paths.HostPaths() docker_paths = paths.DockerPaths() host_tmp_dir = host_paths.tmp_dir host_output_dir = host_paths.output_dir container_args = { "runtime": runtime, "remove": True, "detach": True, "volumes": { host_paths.cwd: { "bind": docker_paths.cwd, "mode": "rw" }, host_paths.dataset_dir: { "bind": docker_paths.dataset_dir, "mode": "rw", }, host_paths.saved_model_dir: { "bind": docker_paths.saved_model_dir, "mode": "rw", }, host_output_dir: { "bind": docker_paths.output_dir, "mode": "rw" }, host_tmp_dir: { "bind": docker_paths.tmp_dir, "mode": "rw" }, }, "shm_size": "16G", } if ports is not None: container_args["ports"] = ports if command is not None: container_args["command"] = command # Windows docker does not require syncronizing file and # directoriy permissions via uid and gid. if os.name != "nt": user_id = os.getuid() group_id = os.getgid() container_args["user"] = f"{user_id}:{group_id}" if envs: container_args["environment"] = envs self.docker_container = self.docker_client.containers.run( image_name, **container_args) logger.info( f"ARMORY Instance {self.docker_container.short_id} created.")
# based on https://github.com/tensorflow/models/tree/master/resnet from __future__ import absolute_import from __future__ import division from __future__ import print_function import numpy as np import tensorflow as tf import json import os import tarfile from armory import paths from art.classifiers import TFClassifier from armory.data.utils import maybe_download_weights_from_s3 SAVED_MODEL_DIR = paths.DockerPaths().saved_model_dir def preprocessing_fn(img): img = img.astype(np.float32) return img class Model(object): """ResNet model.""" def __init__(self, mode): """ResNet constructor. Args: mode: One of 'train' and 'eval'.