def _gather_env_variables(self):
"""
Update the extra env variable dictionary to pass into container or run on host
"""
self.extra_env_vars["ARMORY_GITHUB_TOKEN"] = os.getenv(
"ARMORY_GITHUB_TOKEN", default="")
self.extra_env_vars["ARMORY_PRIVATE_S3_ID"] = os.getenv(
"ARMORY_PRIVATE_S3_ID", default="")
self.extra_env_vars["ARMORY_PRIVATE_S3_KEY"] = os.getenv(
"ARMORY_PRIVATE_S3_KEY", default="")
self.extra_env_vars["ARMORY_INCLUDE_SUBMISSION_BUCKETS"] = os.getenv(
"ARMORY_INCLUDE_SUBMISSION_BUCKETS", default="")
if not self.armory_global_config["verify_ssl"]:
self.extra_env_vars["VERIFY_SSL"] = "false"
if self.config["sysconfig"].get("use_gpu", None):
gpus = self.config["sysconfig"].get("gpus")
if gpus is not None:
self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus
if self.config["sysconfig"].get("set_pythonhashseed"):
self.extra_env_vars["PYTHONHASHSEED"] = "0"
if not self.no_docker:
self.extra_env_vars["HOME"] = "/tmp"
# Because we may want to allow specification of ARMORY_TORCH_HOME
# this constant path is placed here among the other imports
if self.no_docker:
torch_home = paths.HostPaths().pytorch_dir
else:
torch_home = paths.DockerPaths().pytorch_dir
self.extra_env_vars["TORCH_HOME"] = torch_home
self.extra_env_vars[environment.ARMORY_VERSION] = armory.__version__
def __init__(
self,
image_name,
runtime: str = "runc",
envs: dict = None,
ports: dict = None,
command: str = "tail -f /dev/null",
user: str = "",
):
self.docker_client = docker.from_env(version="auto")
host_paths = paths.HostPaths()
docker_paths = paths.DockerPaths()
container_args = {
"runtime": runtime,
"remove": True,
"detach": True,
"volumes": {
host_paths.cwd: {
"bind": docker_paths.cwd,
"mode": "rw"
},
host_paths.dataset_dir: {
"bind": docker_paths.dataset_dir,
"mode": "rw",
},
host_paths.local_git_dir: {
"bind": docker_paths.local_git_dir,
"mode": "rw",
},
host_paths.output_dir: {
"bind": docker_paths.output_dir,
"mode": "rw"
},
host_paths.saved_model_dir: {
"bind": docker_paths.saved_model_dir,
"mode": "rw",
},
host_paths.tmp_dir: {
"bind": docker_paths.tmp_dir,
"mode": "rw"
},
},
"shm_size": "16G",
}
if ports is not None:
container_args["ports"] = ports
if command is not None:
container_args["command"] = command
if user:
container_args["user"] = user
if envs:
container_args["environment"] = envs
self.docker_container = self.docker_client.containers.run(
image_name, **container_args)
logger.info(
f"ARMORY Instance {self.docker_container.short_id} created.")
def split_data(rootdir=None, full="NWPU-RESISC45"):
if rootdir is None:
rootdir = os.path.join(paths.HostPaths().dataset_dir, "downloads",
"manual", "resisc45")
train = "train"
validation = "validation"
test = "test"
for folder in train, validation, test:
for label in LABELS:
os.makedirs(os.path.join(rootdir, folder, label), exist_ok=True)
for label in LABELS:
for i in range(1, 701):
if i <= 500:
split = train
elif i <= 600:
split = validation
else:
split = test
filename = f"{label}_{i:03}.jpg"
source = os.path.join(rootdir, full, label, filename)
target = os.path.join(rootdir, split, label, filename)
print(f"Copying from {source} to {target}")
shutil.copyfile(source, target)
def __init__(
self,
config: dict,
no_docker: bool = False,
root: bool = False,
):
if not isinstance(config, dict):
raise ValueError(f"config {config} must be a dict")
self.config = config
self.host_paths = paths.HostPaths()
if os.path.exists(self.host_paths.armory_config):
self.armory_global_config = load_global_config(
self.host_paths.armory_config)
else:
self.armory_global_config = {"verify_ssl": True}
date_time = datetime.datetime.utcnow().isoformat().replace(":", "")
output_dir = self.config["sysconfig"].get("output_dir", None)
eval_id = f"{output_dir}_{date_time}" if output_dir else date_time
self.config["eval_id"] = eval_id
self.output_dir = os.path.join(self.host_paths.output_dir, eval_id)
self.tmp_dir = os.path.join(self.host_paths.tmp_dir, eval_id)
if self.config["sysconfig"].get("use_gpu", None):
kwargs = dict(runtime="nvidia")
else:
kwargs = dict(runtime="runc")
image_name = self.config["sysconfig"].get("docker_image")
kwargs["image_name"] = image_name
self.no_docker = not image_name or no_docker
self.root = root
# Retrieve environment variables that should be used in evaluation
self.extra_env_vars = dict()
self._gather_env_variables()
if self.no_docker:
if self.root:
raise ValueError(
"running with --root is incompatible with --no-docker")
self.manager = HostManagementInstance()
return
# Download docker image on host
docker_client = docker.from_env()
try:
docker_client.images.get(kwargs["image_name"])
except ImageNotFound:
logger.info(f"Image {image_name} was not found. Downloading...")
if "twosixarmory" in image_name and "-dev" in image_name:
raise ValueError((
"You are attempting to pull an armory developer "
"docker image; however, these are not published. This "
"is likely because you're running armory from its "
"master branch. If you want a stable release with "
"published docker images try pip installing 'armory-testbed' "
"or checking out one of the stable branches on the git repository. "
"If you'd like to continue working on the developer image please "
"build it from source on your machine as described here: "
"https://armory.readthedocs.io/en/latest/contributing/#development-docker-containers"
))
docker_api.pull_verbose(docker_client, image_name)
except requests.exceptions.ConnectionError:
logger.error(
"Docker connection refused. Is Docker Daemon running?")
raise
self.manager = ManagementInstance(**kwargs)
import base64
import os
import pathlib
import shutil
from armory import paths
from armory.utils.external_repo import download_and_extract_repos
HOST_PATHS = paths.HostPaths()
def set_github_token():
"""
Sets a public read-only token to authethenticate our github ci calls. This is only
done because all VMs for GitHub CI share common IP addresses and thus we can get
intermittent rate limits should we not authenticate.
GitHub will revoke the token if it's found in plain-text on a repo. However, this
token does not need to be hidden.
"""
b64_key = b"Njc5MjhkMDA0N2Q5ZTBkNTc4MWNmODgxOGE5ZTVlY2JiOWIzMDg2NQ=="
public_token = base64.b64decode(b64_key).decode()
os.environ["ARMORY_GITHUB_TOKEN"] = public_token
def test_download():
set_github_token()
test_external_repo_dir = pathlib.Path(HOST_PATHS.tmp_dir,
"test-external-repo-subdir")
repo = "twosixlabs/armory-example"
repo_name = repo.split("/")[-1]
def __init__(
self,
image_name,
runtime: str = "runc",
envs: dict = None,
ports: dict = None,
command: str = "tail -f /dev/null",
):
self.docker_client = docker.from_env(version="auto")
host_paths = paths.HostPaths()
docker_paths = paths.DockerPaths()
host_tmp_dir = host_paths.tmp_dir
host_output_dir = host_paths.output_dir
container_args = {
"runtime": runtime,
"remove": True,
"detach": True,
"volumes": {
host_paths.cwd: {
"bind": docker_paths.cwd,
"mode": "rw"
},
host_paths.dataset_dir: {
"bind": docker_paths.dataset_dir,
"mode": "rw",
},
host_paths.saved_model_dir: {
"bind": docker_paths.saved_model_dir,
"mode": "rw",
},
host_output_dir: {
"bind": docker_paths.output_dir,
"mode": "rw"
},
host_tmp_dir: {
"bind": docker_paths.tmp_dir,
"mode": "rw"
},
},
"shm_size": "16G",
}
if ports is not None:
container_args["ports"] = ports
if command is not None:
container_args["command"] = command
# Windows docker does not require syncronizing file and
# directoriy permissions via uid and gid.
if os.name != "nt":
user_id = os.getuid()
group_id = os.getgid()
container_args["user"] = f"{user_id}:{group_id}"
if envs:
container_args["environment"] = envs
self.docker_container = self.docker_client.containers.run(
image_name, **container_args)
logger.info(
f"ARMORY Instance {self.docker_container.short_id} created.")
def download_and_extract_repo(external_repo_name: str,
external_repo_dir: str = None) -> None:
"""
Downloads and extracts an external repository for use within ARMORY. The external
repositories project root will be added to the sys path.
Private repositories require an `ARMORY_GITHUB_TOKEN` environment variable.
:param external_repo_name: String name of "organization/repo-name" or "organization/repo-name@branch"
"""
verify_ssl = get_verify_ssl()
if external_repo_dir is None:
external_repo_dir = paths.HostPaths().external_repo_dir
os.makedirs(external_repo_dir, exist_ok=True)
headers = {}
if "@" in external_repo_name:
org_repo_name, branch = external_repo_name.split("@")
else:
org_repo_name = external_repo_name
branch = "master"
repo_name = org_repo_name.split("/")[-1]
if "ARMORY_GITHUB_TOKEN" in os.environ and os.getenv(
"ARMORY_GITHUB_TOKEN") != "":
headers = {
"Authorization": f'token {os.getenv("ARMORY_GITHUB_TOKEN")}'
}
response = requests.get(
f"https://api.github.com/repos/{org_repo_name}/tarball/{branch}",
headers=headers,
stream=True,
verify=verify_ssl,
)
if response.status_code == 200:
logging.info(f"Downloading external repo: {external_repo_name}")
tar_filename = os.path.join(external_repo_dir, repo_name + ".tar.gz")
with open(tar_filename, "wb") as f:
f.write(response.raw.read())
tar = tarfile.open(tar_filename, "r:gz")
dl_directory_name = tar.getnames()[0]
tar.extractall(path=external_repo_dir)
# Always overwrite existing repositories to keep them at HEAD
final_dir_name = os.path.join(external_repo_dir, repo_name)
if os.path.isdir(final_dir_name):
shutil.rmtree(final_dir_name)
os.rename(
os.path.join(external_repo_dir, dl_directory_name),
final_dir_name,
)
sys.path.insert(0, external_repo_dir)
sys.path.insert(0, final_dir_name)
else:
raise ConnectionError(
"Unable to download repository. If it's private make sure "
"`ARMORY_GITHUB_TOKEN` environment variable is set\n"
f"status_code is {response.status_code}\n"
f"full response is {response.text}")
