def __init__(self):
super().__init__()
if os.path.isfile(self.armory_config):
# Parse paths from config
config = configuration.load_global_config(self.armory_config)
for k in (
"dataset_dir",
"local_git_dir",
"saved_model_dir",
# pytorch_dir should not be found in the config file
# because it is not configurable (yet)
"output_dir",
"tmp_dir",
):
setattr(self, k, config[k])
else:
logger.warning(
f"No {self.armory_config} file. Using default paths.")
logger.warning("Please run `armory configure`")
os.makedirs(self.dataset_dir, exist_ok=True)
os.makedirs(self.local_git_dir, exist_ok=True)
os.makedirs(self.saved_model_dir, exist_ok=True)
os.makedirs(self.pytorch_dir, exist_ok=True)
os.makedirs(self.tmp_dir, exist_ok=True)
os.makedirs(self.output_dir, exist_ok=True)
def __init__(self):
super().__init__()
if os.path.isfile(self.armory_config):
# Parse paths from config
config = configuration.load_global_config(self.armory_config)
for k in (
"dataset_dir",
"saved_model_dir",
"output_dir",
"tmp_dir",
):
setattr(self, k, config[k])
else:
logger.warning(f"No {self.armory_config} file. Using default paths.")
logger.warning("Please run `armory configure`")
os.makedirs(self.dataset_dir, exist_ok=True)
os.makedirs(self.saved_model_dir, exist_ok=True)
os.makedirs(self.tmp_dir, exist_ok=True)
os.makedirs(self.output_dir, exist_ok=True)
def configure(command_args, prog, description):
parser = argparse.ArgumentParser(prog=prog, description=description)
_debug(parser)
args = parser.parse_args(command_args)
coloredlogs.install(level=args.log_level)
default_host_paths = paths.HostDefaultPaths()
config = None
if os.path.exists(default_host_paths.armory_config):
response = None
while response is None:
prompt = f"Existing configuration found: {default_host_paths.armory_config}"
print(prompt)
response = input("Load existing configuration? [Y/n]")
if response in ("Y", "y", ""):
print("Loading configuration...")
config = load_global_config(default_host_paths.armory_config,
validate=False)
print("Load successful")
elif response in ("N", "n"):
print("Configuration not loaded")
else:
print(f"Invalid selection: {response}")
response = None
print()
instructions = "\n".join([
"Configuring paths for armory usage",
f' This configuration will be stored at "{default_host_paths.armory_config}"',
"",
"Please enter desired target directory for the following paths.",
" If left empty, the default path will be used.",
" Absolute paths (which include '~' user paths) are required.",
"",
])
print(instructions)
default_dataset_dir = (config["dataset_dir"] if config is not None
and "dataset_dir" in config.keys() else
default_host_paths.dataset_dir)
default_local_dir = (config["local_git_dir"] if config is not None
and "local_git_dir" in config.keys() else
default_host_paths.local_git_dir)
default_saved_model_dir = (config["saved_model_dir"] if config is not None
and "saved_model_dir" in config.keys() else
default_host_paths.saved_model_dir)
default_tmp_dir = (config["tmp_dir"]
if config is not None and "tmp_dir" in config.keys()
else default_host_paths.tmp_dir)
default_output_dir = (config["output_dir"] if config is not None
and "output_dir" in config.keys() else
default_host_paths.output_dir)
config = {
"dataset_dir": _get_path("dataset_dir", default_dataset_dir),
"local_git_dir": _get_path("local_git_dir", default_local_dir),
"saved_model_dir": _get_path("saved_model_dir",
default_saved_model_dir),
"tmp_dir": _get_path("tmp_dir", default_tmp_dir),
"output_dir": _get_path("output_dir", default_output_dir),
"verify_ssl": _get_verify_ssl(),
}
resolved = "\n".join([
"Resolved paths:",
f" dataset_dir: {config['dataset_dir']}",
f" local_git_dir: {config['local_git_dir']}",
f" saved_model_dir: {config['saved_model_dir']}",
f" tmp_dir: {config['tmp_dir']}",
f" output_dir: {config['output_dir']}",
"Download options:",
f" verify_ssl: {config['verify_ssl']}",
"",
])
print(resolved)
save = None
while save is None:
if os.path.isfile(default_host_paths.armory_config):
print("WARNING: this will overwrite existing configuration.")
print(" Press Ctrl-C to abort.")
answer = input("Save this configuration? [Y/n] ")
if answer in ("Y", "y", ""):
print("Saving configuration...")
save_config(config, default_host_paths.armory_dir)
print("Configure successful")
save = True
elif answer in ("N", "n"):
print("Configuration not saved")
save = False
else:
print(f"Invalid selection: {answer}")
print()
print("Configure complete")
def __init__(
self,
config: dict,
no_docker: bool = False,
root: bool = False,
):
if not isinstance(config, dict):
raise ValueError(f"config {config} must be a dict")
self.config = config
self.host_paths = paths.HostPaths()
if os.path.exists(self.host_paths.armory_config):
self.armory_global_config = load_global_config(
self.host_paths.armory_config)
else:
self.armory_global_config = {"verify_ssl": True}
date_time = datetime.datetime.utcnow().isoformat().replace(":", "")
output_dir = self.config["sysconfig"].get("output_dir", None)
eval_id = f"{output_dir}_{date_time}" if output_dir else date_time
self.config["eval_id"] = eval_id
self.output_dir = os.path.join(self.host_paths.output_dir, eval_id)
self.tmp_dir = os.path.join(self.host_paths.tmp_dir, eval_id)
if self.config["sysconfig"].get("use_gpu", None):
kwargs = dict(runtime="nvidia")
else:
kwargs = dict(runtime="runc")
image_name = self.config["sysconfig"].get("docker_image")
kwargs["image_name"] = image_name
self.no_docker = not image_name or no_docker
self.root = root
# Retrieve environment variables that should be used in evaluation
self.extra_env_vars = dict()
self._gather_env_variables()
if self.no_docker:
if self.root:
raise ValueError(
"running with --root is incompatible with --no-docker")
self.manager = HostManagementInstance()
return
# Download docker image on host
docker_client = docker.from_env()
try:
docker_client.images.get(kwargs["image_name"])
except ImageNotFound:
logger.info(f"Image {image_name} was not found. Downloading...")
if "twosixarmory" in image_name and "-dev" in image_name:
raise ValueError((
"You are attempting to pull an armory developer "
"docker image; however, these are not published. This "
"is likely because you're running armory from its "
"master branch. If you want a stable release with "
"published docker images try pip installing 'armory-testbed' "
"or checking out one of the stable branches on the git repository. "
"If you'd like to continue working on the developer image please "
"build it from source on your machine as described here: "
"https://armory.readthedocs.io/en/latest/contributing/#development-docker-containers"
))
docker_api.pull_verbose(docker_client, image_name)
except requests.exceptions.ConnectionError:
logger.error(
"Docker connection refused. Is Docker Daemon running?")
raise
self.manager = ManagementInstance(**kwargs)