def _gather_env_variables(self): """ Update the extra env variable dictionary to pass into container or run on host """ self.extra_env_vars["ARMORY_GITHUB_TOKEN"] = os.getenv( "ARMORY_GITHUB_TOKEN", default="") self.extra_env_vars["ARMORY_PRIVATE_S3_ID"] = os.getenv( "ARMORY_PRIVATE_S3_ID", default="") self.extra_env_vars["ARMORY_PRIVATE_S3_KEY"] = os.getenv( "ARMORY_PRIVATE_S3_KEY", default="") self.extra_env_vars["ARMORY_INCLUDE_SUBMISSION_BUCKETS"] = os.getenv( "ARMORY_INCLUDE_SUBMISSION_BUCKETS", default="") if not self.armory_global_config["verify_ssl"]: self.extra_env_vars["VERIFY_SSL"] = "false" if self.config["sysconfig"].get("use_gpu", None): gpus = self.config["sysconfig"].get("gpus") if gpus is not None: self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus if self.config["sysconfig"].get("set_pythonhashseed"): self.extra_env_vars["PYTHONHASHSEED"] = "0" if not self.no_docker: self.extra_env_vars["HOME"] = "/tmp" # Because we may want to allow specification of ARMORY_TORCH_HOME # this constant path is placed here among the other imports if self.no_docker: torch_home = paths.HostPaths().pytorch_dir else: torch_home = paths.DockerPaths().pytorch_dir self.extra_env_vars["TORCH_HOME"] = torch_home self.extra_env_vars[environment.ARMORY_VERSION] = armory.__version__
def __init__( self, image_name, runtime: str = "runc", envs: dict = None, ports: dict = None, command: str = "tail -f /dev/null", user: str = "", ): self.docker_client = docker.from_env(version="auto") host_paths = paths.HostPaths() docker_paths = paths.DockerPaths() container_args = { "runtime": runtime, "remove": True, "detach": True, "volumes": { host_paths.cwd: { "bind": docker_paths.cwd, "mode": "rw" }, host_paths.dataset_dir: { "bind": docker_paths.dataset_dir, "mode": "rw", }, host_paths.local_git_dir: { "bind": docker_paths.local_git_dir, "mode": "rw", }, host_paths.output_dir: { "bind": docker_paths.output_dir, "mode": "rw" }, host_paths.saved_model_dir: { "bind": docker_paths.saved_model_dir, "mode": "rw", }, host_paths.tmp_dir: { "bind": docker_paths.tmp_dir, "mode": "rw" }, }, "shm_size": "16G", } if ports is not None: container_args["ports"] = ports if command is not None: container_args["command"] = command if user: container_args["user"] = user if envs: container_args["environment"] = envs self.docker_container = self.docker_client.containers.run( image_name, **container_args) logger.info( f"ARMORY Instance {self.docker_container.short_id} created.")
def split_data(rootdir=None, full="NWPU-RESISC45"): if rootdir is None: rootdir = os.path.join(paths.HostPaths().dataset_dir, "downloads", "manual", "resisc45") train = "train" validation = "validation" test = "test" for folder in train, validation, test: for label in LABELS: os.makedirs(os.path.join(rootdir, folder, label), exist_ok=True) for label in LABELS: for i in range(1, 701): if i <= 500: split = train elif i <= 600: split = validation else: split = test filename = f"{label}_{i:03}.jpg" source = os.path.join(rootdir, full, label, filename) target = os.path.join(rootdir, split, label, filename) print(f"Copying from {source} to {target}") shutil.copyfile(source, target)
def __init__( self, config: dict, no_docker: bool = False, root: bool = False, ): if not isinstance(config, dict): raise ValueError(f"config {config} must be a dict") self.config = config self.host_paths = paths.HostPaths() if os.path.exists(self.host_paths.armory_config): self.armory_global_config = load_global_config( self.host_paths.armory_config) else: self.armory_global_config = {"verify_ssl": True} date_time = datetime.datetime.utcnow().isoformat().replace(":", "") output_dir = self.config["sysconfig"].get("output_dir", None) eval_id = f"{output_dir}_{date_time}" if output_dir else date_time self.config["eval_id"] = eval_id self.output_dir = os.path.join(self.host_paths.output_dir, eval_id) self.tmp_dir = os.path.join(self.host_paths.tmp_dir, eval_id) if self.config["sysconfig"].get("use_gpu", None): kwargs = dict(runtime="nvidia") else: kwargs = dict(runtime="runc") image_name = self.config["sysconfig"].get("docker_image") kwargs["image_name"] = image_name self.no_docker = not image_name or no_docker self.root = root # Retrieve environment variables that should be used in evaluation self.extra_env_vars = dict() self._gather_env_variables() if self.no_docker: if self.root: raise ValueError( "running with --root is incompatible with --no-docker") self.manager = HostManagementInstance() return # Download docker image on host docker_client = docker.from_env() try: docker_client.images.get(kwargs["image_name"]) except ImageNotFound: logger.info(f"Image {image_name} was not found. Downloading...") if "twosixarmory" in image_name and "-dev" in image_name: raise ValueError(( "You are attempting to pull an armory developer " "docker image; however, these are not published. This " "is likely because you're running armory from its " "master branch. If you want a stable release with " "published docker images try pip installing 'armory-testbed' " "or checking out one of the stable branches on the git repository. " "If you'd like to continue working on the developer image please " "build it from source on your machine as described here: " "https://armory.readthedocs.io/en/latest/contributing/#development-docker-containers" )) docker_api.pull_verbose(docker_client, image_name) except requests.exceptions.ConnectionError: logger.error( "Docker connection refused. Is Docker Daemon running?") raise self.manager = ManagementInstance(**kwargs)
import base64 import os import pathlib import shutil from armory import paths from armory.utils.external_repo import download_and_extract_repos HOST_PATHS = paths.HostPaths() def set_github_token(): """ Sets a public read-only token to authethenticate our github ci calls. This is only done because all VMs for GitHub CI share common IP addresses and thus we can get intermittent rate limits should we not authenticate. GitHub will revoke the token if it's found in plain-text on a repo. However, this token does not need to be hidden. """ b64_key = b"Njc5MjhkMDA0N2Q5ZTBkNTc4MWNmODgxOGE5ZTVlY2JiOWIzMDg2NQ==" public_token = base64.b64decode(b64_key).decode() os.environ["ARMORY_GITHUB_TOKEN"] = public_token def test_download(): set_github_token() test_external_repo_dir = pathlib.Path(HOST_PATHS.tmp_dir, "test-external-repo-subdir") repo = "twosixlabs/armory-example" repo_name = repo.split("/")[-1]
def __init__( self, image_name, runtime: str = "runc", envs: dict = None, ports: dict = None, command: str = "tail -f /dev/null", ): self.docker_client = docker.from_env(version="auto") host_paths = paths.HostPaths() docker_paths = paths.DockerPaths() host_tmp_dir = host_paths.tmp_dir host_output_dir = host_paths.output_dir container_args = { "runtime": runtime, "remove": True, "detach": True, "volumes": { host_paths.cwd: { "bind": docker_paths.cwd, "mode": "rw" }, host_paths.dataset_dir: { "bind": docker_paths.dataset_dir, "mode": "rw", }, host_paths.saved_model_dir: { "bind": docker_paths.saved_model_dir, "mode": "rw", }, host_output_dir: { "bind": docker_paths.output_dir, "mode": "rw" }, host_tmp_dir: { "bind": docker_paths.tmp_dir, "mode": "rw" }, }, "shm_size": "16G", } if ports is not None: container_args["ports"] = ports if command is not None: container_args["command"] = command # Windows docker does not require syncronizing file and # directoriy permissions via uid and gid. if os.name != "nt": user_id = os.getuid() group_id = os.getgid() container_args["user"] = f"{user_id}:{group_id}" if envs: container_args["environment"] = envs self.docker_container = self.docker_client.containers.run( image_name, **container_args) logger.info( f"ARMORY Instance {self.docker_container.short_id} created.")
def download_and_extract_repo(external_repo_name: str, external_repo_dir: str = None) -> None: """ Downloads and extracts an external repository for use within ARMORY. The external repositories project root will be added to the sys path. Private repositories require an `ARMORY_GITHUB_TOKEN` environment variable. :param external_repo_name: String name of "organization/repo-name" or "organization/repo-name@branch" """ verify_ssl = get_verify_ssl() if external_repo_dir is None: external_repo_dir = paths.HostPaths().external_repo_dir os.makedirs(external_repo_dir, exist_ok=True) headers = {} if "@" in external_repo_name: org_repo_name, branch = external_repo_name.split("@") else: org_repo_name = external_repo_name branch = "master" repo_name = org_repo_name.split("/")[-1] if "ARMORY_GITHUB_TOKEN" in os.environ and os.getenv( "ARMORY_GITHUB_TOKEN") != "": headers = { "Authorization": f'token {os.getenv("ARMORY_GITHUB_TOKEN")}' } response = requests.get( f"https://api.github.com/repos/{org_repo_name}/tarball/{branch}", headers=headers, stream=True, verify=verify_ssl, ) if response.status_code == 200: logging.info(f"Downloading external repo: {external_repo_name}") tar_filename = os.path.join(external_repo_dir, repo_name + ".tar.gz") with open(tar_filename, "wb") as f: f.write(response.raw.read()) tar = tarfile.open(tar_filename, "r:gz") dl_directory_name = tar.getnames()[0] tar.extractall(path=external_repo_dir) # Always overwrite existing repositories to keep them at HEAD final_dir_name = os.path.join(external_repo_dir, repo_name) if os.path.isdir(final_dir_name): shutil.rmtree(final_dir_name) os.rename( os.path.join(external_repo_dir, dl_directory_name), final_dir_name, ) sys.path.insert(0, external_repo_dir) sys.path.insert(0, final_dir_name) else: raise ConnectionError( "Unable to download repository. If it's private make sure " "`ARMORY_GITHUB_TOKEN` environment variable is set\n" f"status_code is {response.status_code}\n" f"full response is {response.text}")