def _pull_docker_images(docker_client=None):
if docker_client is None:
docker_client = docker.from_env(version="auto")
for image in images.ALL:
try:
docker_client.images.get(image)
except ImageNotFound:
if armory.is_dev():
raise ValueError(
"For '-dev', please run 'docker/build-dev.sh' locally before running armory"
)
logger.info(f"Image {image} was not found. Downloading...")
docker_api.pull_verbose(docker_client, image)
def _pull_docker_images(docker_client=None):
if docker_client is None:
docker_client = docker.from_env(version="auto")
for image in images.ALL:
try:
docker_client.images.get(image)
except docker.errors.ImageNotFound:
try:
logger.info(f"Image {image} was not found. Downloading...")
docker_api.pull_verbose(docker_client, image)
except docker.errors.NotFound:
logger.exception(
f"Docker image {image} does not exist for this version. "
f"Please run 'bash docker/build.sh {image}'"
"or 'bash docker/build.sh all' before running armory")
raise ValueError(f"Docker image {image} not built")
def __init__(
self,
config: dict,
no_docker: bool = False,
root: bool = False,
):
if not isinstance(config, dict):
raise ValueError(f"config {config} must be a dict")
self.config = config
self.host_paths = paths.HostPaths()
if os.path.exists(self.host_paths.armory_config):
self.armory_global_config = load_global_config(
self.host_paths.armory_config)
else:
self.armory_global_config = {"verify_ssl": True}
date_time = datetime.datetime.utcnow().isoformat().replace(":", "")
output_dir = self.config["sysconfig"].get("output_dir", None)
eval_id = f"{output_dir}_{date_time}" if output_dir else date_time
self.config["eval_id"] = eval_id
self.output_dir = os.path.join(self.host_paths.output_dir, eval_id)
self.tmp_dir = os.path.join(self.host_paths.tmp_dir, eval_id)
if self.config["sysconfig"].get("use_gpu", None):
kwargs = dict(runtime="nvidia")
else:
kwargs = dict(runtime="runc")
image_name = self.config["sysconfig"].get("docker_image")
kwargs["image_name"] = image_name
self.no_docker = not image_name or no_docker
self.root = root
# Retrieve environment variables that should be used in evaluation
self.extra_env_vars = dict()
self._gather_env_variables()
if self.no_docker:
if self.root:
raise ValueError(
"running with --root is incompatible with --no-docker")
self.manager = HostManagementInstance()
return
# Download docker image on host
docker_client = docker.from_env()
try:
docker_client.images.get(kwargs["image_name"])
except ImageNotFound:
logger.info(f"Image {image_name} was not found. Downloading...")
if "twosixarmory" in image_name and "-dev" in image_name:
raise ValueError((
"You are attempting to pull an armory developer "
"docker image; however, these are not published. This "
"is likely because you're running armory from its "
"master branch. If you want a stable release with "
"published docker images try pip installing 'armory-testbed' "
"or checking out one of the stable branches on the git repository. "
"If you'd like to continue working on the developer image please "
"build it from source on your machine as described here: "
"https://armory.readthedocs.io/en/latest/contributing/#development-docker-containers"
))
docker_api.pull_verbose(docker_client, image_name)
except requests.exceptions.ConnectionError:
logger.error(
"Docker connection refused. Is Docker Daemon running?")
raise
self.manager = ManagementInstance(**kwargs)
def __init__(self,
config_path: Union[str, dict],
container_config_name="eval-config.json"):
self.host_paths = paths.host()
self.docker_paths = paths.docker()
if os.name != "nt":
self.user_id, self.group_id = os.getuid(), os.getgid()
else:
self.user_id, self.group_id = 0, 0
self.extra_env_vars = dict()
if isinstance(config_path, str):
try:
self.config = load_config(config_path)
except json.decoder.JSONDecodeError:
logger.error(f"Could not decode {config_path} as a json file.")
if not config_path.lower().endswith(".json"):
logger.warning(f"{config_path} is not a '*.json' file")
logger.warning(
"If using `armory run`, use a json config file.")
raise
elif isinstance(config_path, dict):
self.config = config_path
else:
raise ValueError(
f"config_path {config_path} must be a str or dict")
(
self.container_subdir,
self.tmp_dir,
self.output_dir,
) = volumes_util.tmp_output_subdir()
self.tmp_config = os.path.join(self.tmp_dir, container_config_name)
self.external_repo_dir = paths.get_external(self.tmp_dir)
self.docker_config_path = Path(
os.path.join(self.docker_paths.tmp_dir,
container_config_name)).as_posix()
kwargs = dict(runtime="runc")
if self.config["sysconfig"].get("use_gpu", None):
kwargs["runtime"] = "nvidia"
gpus = self.config["sysconfig"].get("gpus")
if gpus is not None:
self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus
if self.config["sysconfig"].get("external_github_repo", None):
self._download_external()
self.extra_env_vars.update(
{"PYTHONPATH": self.docker_paths.external_repo_dir})
if self.config["sysconfig"].get("use_armory_private", None):
self._download_private()
image_name = self.config["sysconfig"].get("docker_image")
kwargs["image_name"] = image_name
# Download docker image on host
docker_client = docker.from_env()
try:
docker_client.images.get(kwargs["image_name"])
except ImageNotFound:
logger.info(f"Image {image_name} was not found. Downloading...")
docker_api.pull_verbose(docker_client, image_name)
except requests.exceptions.ConnectionError:
logger.error(
f"Docker connection refused. Is Docker Daemon running?")
raise
self.manager = ManagementInstance(**kwargs)