def download(command_args, prog, description): """ Script to download all datasets and model weights for offline usage. """ parser = argparse.ArgumentParser(prog=prog, description=description) _debug(parser) parser.add_argument( metavar="<download data config file>", dest="download_config", type=str, action=DownloadConfig, help= f"Configuration for download of data. See {DEFAULT_SCENARIO}. Note: file must be under current working directory.", ) parser.add_argument( metavar="<scenario>", dest="scenario", type=str, default="all", help= "scenario for which to download data, 'list' for available scenarios, or blank to download all scenarios", nargs="?", ) _no_docker(parser) args = parser.parse_args(command_args) coloredlogs.install(level=args.log_level) if args.no_docker: logger.info( "Downloading requested datasets and model weights in host mode...") paths.set_mode("host") from armory.data import datasets from armory.data import model_weights datasets.download_all(args.download_config, args.scenario) model_weights.download_all(args.download_config, args.scenario) return if not armory.is_dev(): logger.info("Downloading all docker images....") _pull_docker_images() logger.info("Downloading requested datasets and model weights...") config = {"sysconfig": {"docker_image": images.TF1}} rig = Evaluator(config) cmd = "; ".join([ "import logging", "import coloredlogs", f"coloredlogs.install({args.log_level})", "from armory.data import datasets", "from armory.data import model_weights", f'datasets.download_all("{args.download_config}", "{args.scenario}")', f'model_weights.download_all("{args.download_config}", "{args.scenario}")', ]) exit_code = rig.run(command=f"python -c '{cmd}'") sys.exit(exit_code)
def _pull_docker_images(docker_client=None): if docker_client is None: docker_client = docker.from_env(version="auto") for image in images.ALL: try: docker_client.images.get(image) except ImageNotFound: if armory.is_dev(): raise ValueError( "For '-dev', please run 'docker/build-dev.sh' locally before running armory" ) logger.info(f"Image {image} was not found. Downloading...") docker_api.pull_verbose(docker_client, image)
def download(command_args, prog, description): """ Script to download all datasets and model weights for offline usage. """ parser = argparse.ArgumentParser(prog=prog, description=description) parser.add_argument( "-d", "--debug", dest="log_level", action="store_const", const=logging.DEBUG, default=logging.INFO, help="Debug output (logging=DEBUG)", ) parser.add_argument( metavar="<download data config file>", dest="download_config", type=str, action=DownloadConfig, help= f"Configuration for download of data. See {DEFAULT_SCENARIO}. Note: file must be under current working directory.", ) parser.add_argument( metavar="<scenario>", dest="scenario", type=str, default="all", help= "scenario for which to download data, 'list' for available scenarios, or blank to download all scenarios", nargs="?", ) try: args = parser.parse_args(command_args) except SystemExit: parser.print_help() raise coloredlogs.install(level=args.log_level) paths.host() if not armory.is_dev(): print("Downloading all docker images....") _pull_docker_images() print("Downloading requested datasets and model weights...") manager = ManagementInstance(image_name=images.TF1) runner = manager.start_armory_instance() cmd = "; ".join([ "import logging", "import coloredlogs", "coloredlogs.install(logging.INFO)", "from armory.data import datasets", "from armory.data import model_weights", f'datasets.download_all("{args.download_config}", "{args.scenario}")', f'model_weights.download_all("{args.download_config}", "{args.scenario}")', ]) runner.exec_cmd(f"python -c '{cmd}'") manager.stop_armory_instance(runner)