def ensure_armory_dirs(request):
"""
CI doesn't mount volumes
"""
saved_model_dir = paths.docker().saved_model_dir
dataset_dir = paths.docker().dataset_dir
output_dir = paths.docker().output_dir
os.makedirs(saved_model_dir, exist_ok=True)
os.makedirs(dataset_dir, exist_ok=True)
os.makedirs(output_dir, exist_ok=True)
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = ResNet50(weights=None, **model_kwargs)
if weights_file:
saved_model_dir = paths.docker().saved_model_dir
filepath = os.path.join(saved_model_dir, weights_file)
if not os.path.isfile(filepath):
download_file_from_s3(
"armory-public-data",
f"model-weights/{weights_file}",
f"{saved_model_dir}/{weights_file}",
)
model.load_weights(filepath)
wrapped_model = KerasClassifier(
model,
clip_values=(
np.array([
0.0 - IMAGENET_MEANS[0],
0.0 - IMAGENET_MEANS[1],
0.0 - IMAGENET_MEANS[2],
]),
np.array([
255.0 - IMAGENET_MEANS[0],
255.0 - IMAGENET_MEANS[1],
255.0 - IMAGENET_MEANS[2],
]),
),
**wrapper_kwargs,
)
return wrapped_model
def save(self, config: dict, results: dict, adv_examples=None):
"""
Saves a results json-formattable output to file
adv_examples are (optional) instances of the actual examples used.
It will be saved in a binary format.
"""
if adv_examples is not None:
raise NotImplementedError("saving adversarial examples")
scenario_name = config["scenario"]["name"]
filename = f"{scenario_name}_{int(time.time())}.json"
logger.info(
f"Saving evaluation results saved to <output_dir>/{filename}")
with open(os.path.join(paths.docker().output_dir, filename), "w") as f:
output_dict = {
"armory_version": armory.__version__,
"config": config,
"results": results,
}
f.write(json.dumps(output_dict, sort_keys=True, indent=4) + "\n")
def maybe_download_weights_from_s3(weights_file: str) -> str:
"""
:param weights_file:
:return:
"""
saved_model_dir = paths.docker().saved_model_dir
filepath = os.path.join(saved_model_dir, weights_file)
if os.path.isfile(filepath):
logger.info(f"Using available {weights_file} in Armory `saved_model_dir`")
else:
logger.info(
f"{weights_file} not found in Armory `saved_model_dir`. Attempting to pull weights from S3"
)
download_file_from_s3(
"armory-public-data",
f"model-weights/{weights_file}",
f"{saved_model_dir}/{weights_file}",
)
return filepath
from importlib import import_module
import numpy as np
import pytest
from armory.data import datasets
from armory import paths
DATASET_DIR = paths.docker().dataset_dir
@pytest.mark.usefixtures("ensure_armory_dirs")
def test_keras_mnist():
classifier_module = import_module("armory.baseline_models.keras.mnist")
classifier_fn = getattr(classifier_module, "get_art_model")
classifier = classifier_fn(model_kwargs={}, wrapper_kwargs={})
preprocessing_fn = getattr(classifier_module, "preprocessing_fn")
train_dataset = datasets.mnist(
split_type="train",
epochs=1,
batch_size=600,
dataset_dir=DATASET_DIR,
preprocessing_fn=preprocessing_fn,
)
test_dataset = datasets.mnist(
split_type="test",
epochs=1,
batch_size=100,
dataset_dir=DATASET_DIR,
preprocessing_fn=preprocessing_fn,
def __init__(self,
config_path: Union[str, dict],
container_config_name="eval-config.json"):
self.host_paths = paths.host()
self.docker_paths = paths.docker()
if os.name != "nt":
self.user_id, self.group_id = os.getuid(), os.getgid()
else:
self.user_id, self.group_id = 0, 0
self.extra_env_vars = dict()
if isinstance(config_path, str):
try:
self.config = load_config(config_path)
except json.decoder.JSONDecodeError:
logger.error(f"Could not decode {config_path} as a json file.")
if not config_path.lower().endswith(".json"):
logger.warning(f"{config_path} is not a '*.json' file")
logger.warning(
"If using `armory run`, use a json config file.")
raise
elif isinstance(config_path, dict):
self.config = config_path
else:
raise ValueError(
f"config_path {config_path} must be a str or dict")
(
self.container_subdir,
self.tmp_dir,
self.output_dir,
) = volumes_util.tmp_output_subdir()
self.tmp_config = os.path.join(self.tmp_dir, container_config_name)
self.external_repo_dir = paths.get_external(self.tmp_dir)
self.docker_config_path = Path(
os.path.join(self.docker_paths.tmp_dir,
container_config_name)).as_posix()
kwargs = dict(runtime="runc")
if self.config["sysconfig"].get("use_gpu", None):
kwargs["runtime"] = "nvidia"
gpus = self.config["sysconfig"].get("gpus")
if gpus is not None:
self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus
if self.config["sysconfig"].get("external_github_repo", None):
self._download_external()
self.extra_env_vars.update(
{"PYTHONPATH": self.docker_paths.external_repo_dir})
if self.config["sysconfig"].get("use_armory_private", None):
self._download_private()
image_name = self.config["sysconfig"].get("docker_image")
kwargs["image_name"] = image_name
# Download docker image on host
docker_client = docker.from_env()
try:
docker_client.images.get(kwargs["image_name"])
except ImageNotFound:
logger.info(f"Image {image_name} was not found. Downloading...")
docker_api.pull_verbose(docker_client, image_name)
except requests.exceptions.ConnectionError:
logger.error(
f"Docker connection refused. Is Docker Daemon running?")
raise
self.manager = ManagementInstance(**kwargs)
def _generator_from_tfds(
dataset_name: str,
split_type: str,
batch_size: int,
epochs: int,
dataset_dir: str,
preprocessing_fn: Callable,
as_supervised: bool = True,
supervised_xy_keys=None,
download_and_prepare_kwargs=None,
variable_length=False,
shuffle_files=True,
cache_dataset: bool = True,
):
"""
If as_supervised=False, must designate keys as a tuple in supervised_xy_keys:
supervised_xy_keys=('video', 'label') # ucf101 dataset
if variable_length=True and batch_size > 1:
output batches are 1D np.arrays of objects
"""
if not dataset_dir:
dataset_dir = paths.docker().dataset_dir
if cache_dataset:
_cache_dataset(
dataset_dir,
dataset_name=dataset_name,
)
default_graph = tf.compat.v1.keras.backend.get_session().graph
ds, ds_info = tfds.load(
dataset_name,
split=split_type,
as_supervised=as_supervised,
data_dir=dataset_dir,
with_info=True,
download_and_prepare_kwargs=download_and_prepare_kwargs,
shuffle_files=shuffle_files,
)
if not as_supervised:
try:
x_key, y_key = supervised_xy_keys
except (TypeError, ValueError):
raise ValueError(
f"When as_supervised=False, supervised_xy_keys must be a (x_key, y_key)"
f" tuple, not {supervised_xy_keys}")
if not isinstance(x_key, str) or not isinstance(y_key, str):
raise ValueError(f"supervised_xy_keys be a tuple of strings,"
f" not {type(x_key), type(y_key)}")
ds = ds.map(lambda x: (x[x_key], x[y_key]))
ds = ds.repeat(epochs)
if shuffle_files:
ds = ds.shuffle(batch_size * 10, reshuffle_each_iteration=True)
if variable_length and batch_size > 1:
ds = ds.batch(1, drop_remainder=False)
else:
ds = ds.batch(batch_size, drop_remainder=False)
ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
ds = tfds.as_numpy(ds, graph=default_graph)
generator = ArmoryDataGenerator(
ds,
size=ds_info.splits[split_type].num_examples,
batch_size=batch_size,
epochs=epochs,
preprocessing_fn=preprocessing_fn,
variable_length=bool(variable_length and batch_size > 1),
)
return generator