class DockerDevice(object): """A Docker Device is capable of creating and launching docker images. In order to successfully create and launch a docker image you must either run this as root, or have enabled sudoless docker. """ TAG_REGEX = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]*:?[a-zA-Z0-9._-]*") GPU_BASEIMG = ( "FROM nvidia/opengl:1.0-glvnd-runtime-ubuntu18.04 AS emulator\n" + "ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display") DEFAULT_BASE_IMG = "FROM debian:stretch-slim AS emulator" def __init__(self, emulator, sysimg, dest_dir, gpu=False, repo="", tag="", name=""): self.sysimg = AndroidReleaseZip(sysimg) self.emulator = AndroidReleaseZip(emulator) self.dest = dest_dir self.writer = TemplateWriter(dest_dir) if repo and repo[-1] != "/": repo += "/" if not name: name = self.sysimg.repo_friendly_name() repo += name if gpu: repo += "-gpu" if not tag: tag = self.emulator.build_id() self.tag = "{}:{}".format(repo, tag) if not self.TAG_REGEX.match(self.tag): raise Exception("The resulting tag: {} is not a valid docker tag.", self.tag) # The following are only set after creating/launching. self.container = None self.identity = None self.base_img = DockerDevice.DEFAULT_BASE_IMG if gpu: self.base_img = DockerDevice.GPU_BASEIMG def push(self, sha, latest=False): repo, tag = self.tag.rsplit(":", 1) print("Pushing docker image: {}.. be patient this can take a while!". format(self.tag)) tracker = ProgressTracker() try: client = docker.from_env() result = client.images.push(repo, tag, stream=True, decode=True) for entry in result: tracker.update(entry) except: logging.exception("Failed to push image.", exc_info=True) logging.warning("You can manually push the image as follows:") logging.warning("docker push {}".format(self.tag)) def _copy_adb_to(self, dest): """Find adb, or download it if needed.""" logging.info("Retrieving platform-tools") tools = PlatformTools() tools.extract_adb(dest) def _read_adb_key(self): adb_path = os.path.expanduser("~/.android/adbkey") if os.path.exists(adb_path): with open(adb_path, "r") as adb: return adb.read() return "" def get_api_client(self): try: api_client = docker.APIClient() logging.info(api_client.version()) return api_client except: logging.exception( "Failed to create default client, trying domain socket.", exc_info=True) api_client = docker.APIClient(base_url="unix://var/run/docker.sock") logging.info(api_client.version()) return api_client def create_container(self): """Creates the docker container, returning the sha of the container, or None in case of failure.""" identity = None print("Creating docker image: {}.. be patient this can take a while!". format(self.tag)) try: logging.info("build(path=%s, tag=%s, rm=True, decode=True)", self.dest, self.tag) api_client = self.get_api_client() result = api_client.build(path=self.dest, tag=self.tag, rm=True, decode=True) for entry in result: if "stream" in entry: sys.stdout.write(entry["stream"]) if "aux" in entry and "ID" in entry["aux"]: identity = entry["aux"]["ID"] except: logging.exception("Failed to create container.", exc_info=True) logging.warning( "You can manually create the container as follows:") logging.warning("docker build {}".format(self.dest)) self.identity = identity return identity def create_cloud_build_step(self): return { "name": "gcr.io/cloud-builders/docker", "args": ["build", "-t", self.tag, os.path.basename(self.dest)] } def launch(self, image_sha, port=5555): """Launches the container with the given sha, publishing abd on port, and grpc on port 8554 Returns the container. """ client = docker.from_env() try: container = client.containers.run( image=image_sha, privileged=True, publish_all_ports=True, detach=True, ports={ "5555/tcp": port, "8554/tcp": 8554 }, environment={"ADBKEY": self._read_adb_key()}, ) self.container = container print("Launched {} (id:{})".format(container.name, container.id)) print("docker logs -f {}".format(container.name)) print("docker stop {}".format(container.name)) return container except: logging.exception("Unable to run the %s", image_sha) print("Unable to start the container, try running it as:") print("./run.sh {}", image_sha) def bin_place_emulator_files(self, by_copying_zip_files): """Bin places the emulator files for the docker file.""" if by_copying_zip_files: logging.info("Copying zips to docker src dir: %s", self.dest) shutil.copy2(self.emulator.fname, self.dest) shutil.copy2(self.sysimg.fname, self.dest) logging.info("Done copying") else: logging.info("Unzipping zips to docker src dir: %s", self.dest) extract_zip(self.emulator.fname, os.path.join(self.dest, "emu")) extract_zip(self.sysimg.fname, os.path.join(self.dest, "sys")) logging.info("Done unzipping") def create_docker_file(self, extra="", metrics=False, by_copying_zip_files=False, screen_resolution=None): if screen_resolution is None: screen_resolution = DEFAULT_RESOLUTION if type(extra) is list: extra = " ".join(extra) logging.info("Emulator zip: %s", self.emulator) logging.info("Sysimg zip: %s", self.sysimg) logging.info("Docker src dir: %s", self.dest) date = datetime.datetime.utcnow().isoformat("T") + "Z" # Make sure the destination directory is empty. if os.path.exists(self.dest): shutil.rmtree(self.dest) mkdir_p(self.dest) self._copy_adb_to(self.dest) self.writer.write_template("avd/Pixel2.ini", {"api": self.sysimg.api()}) self.writer.write_template( "avd/Pixel2.avd/config.ini", { "playstore": self.sysimg.tag() == "google_apis_playstore", "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "tag": self.sysimg.tag(), 'width': screen_resolution['width'], 'height': screen_resolution['height'], 'density': screen_resolution['density'] }, ) metrics_msg = NO_METRICS_MESSAGE # Only version 29.3.1 >= can collect metrics. if metrics and version.parse( self.emulator.revision()) >= version.parse("29.3.1"): extra += " -metrics-collection" metrics_msg = METRICS_MESSAGE # Include a README.MD message. self.writer.write_template( "emulator.README.MD", { "metrics": metrics_msg, "dessert": self.sysimg.codename(), "tag": self.sysimg.tag(), "container_id": self.tag, "emu_build_id": self.emulator.build_id(), }, rename_as="README.MD", ) extra += " {}".format(self.sysimg.logger_flags()) self.writer.write_template("launch-emulator.sh", { "extra": extra, "version": emu.__version__ }) self.writer.write_template("default.pa", {}) self.bin_place_emulator_files(by_copying_zip_files) src_template = "Dockerfile.from_zip" if by_copying_zip_files else "Dockerfile" self.writer.write_template( src_template, { "user": "******".format(os.environ.get("USER", "unknown"), socket.gethostname()), "tag": self.sysimg.tag(), "api": self.sysimg.api(), "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "gpu": self.sysimg.gpu(), "emu_zip": os.path.basename(self.emulator.fname), "emu_build_id": self.emulator.build_id(), "sysimg_zip": os.path.basename(self.sysimg.fname), "date": date, "from_base_img": self.base_img, }, rename_as="Dockerfile", )
class DockerDevice(object): """A Docker Device is capable of creating and launching docker images. In order to successfully create and launch a docker image you must either run this as root, or have enabled sudoless docker. """ TAG_REGEX = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]*:?[a-zA-Z0-9._-]*") GPU_BASEIMG = ( "FROM nvidia/opengl:1.0-glvnd-runtime-ubuntu18.04 AS emulator\n" + "ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display") DEFAULT_BASE_IMG = "FROM debian:stretch-slim AS emulator" def __init__(self, emulator, sysimg, dest_dir, gpu=False, repo="", tag=""): self.sysimg = AndroidReleaseZip(sysimg) self.emulator = AndroidReleaseZip(emulator) self.dest = dest_dir self.env = Environment(loader=PackageLoader("emu", "templates")) if repo and repo[-1] != "/": repo += "/" repo += self.sysimg.repo_friendly_name() if gpu: repo += "-gpu" if not tag: tag = self.emulator.build_id() self.tag = "{}:{}".format(repo, tag) if not self.TAG_REGEX.match(self.tag): raise Exception("The resulting tag: {} is not a valid docker tag.", self.tag) # The following are only set after creating/launching. self.container = None self.identity = None self.base_img = DockerDevice.DEFAULT_BASE_IMG if gpu: self.base_img = DockerDevice.GPU_BASEIMG def push(self, sha, latest=False): repo, tag = self.tag.split(":") print("Pushing docker image: {}.. be patient this can take a while!". format(self.tag)) tracker = ProgressTracker() try: client = docker.from_env() result = client.images.push(repo, tag, stream=True, decode=True) for entry in result: tracker.update(entry) except: logging.exception("Failed to push image.", exc_info=True) logging.warning("You can manually push the image as follows:") logging.warning("docker push {}".format(self.tag)) def _copy_adb_to(self, dest): """Find adb, or download it if needed.""" logging.info("Retrieving platform-tools") tools = PlatformTools() tools.extract_adb(dest) def _read_adb_key(self): adb_path = os.path.expanduser("~/.android/adbkey") if os.path.exists(adb_path): with open(adb_path, "r") as adb: return adb.read() return "" def _write_template(self, tmpl_file, template_dict): """Loads the the given template, writing it out to the dest_dir. Note: the template will be written {dest_dir}/{tmpl_file}, directories will be created if the do not yet exist. """ dest = os.path.join(self.dest, tmpl_file) dest_dir = os.path.dirname(dest) mkdir_p(dest_dir) logging.info("Writing: %s with %s", dest, template_dict) template = self.env.get_template(tmpl_file) with open(dest, "w") as dfile: dfile.write(template.render(template_dict)) def get_api_client(self): try: api_client = docker.APIClient() logging.info(api_client.version()) return api_client except: logging.exception( "Failed to create default client, trying domain socket.", exc_info=True) api_client = docker.APIClient(base_url="unix://var/run/docker.sock") logging.info(api_client.version()) return api_client def create_container(self): """Creates the docker container, returning the sha of the container, or None in case of failure.""" identity = None print("Creating docker image: {}.. be patient this can take a while!". format(self.tag)) try: logging.info("build(path=%s, tag=%s, rm=True, decode=True)", self.dest, self.tag) api_client = self.get_api_client() result = api_client.build(path=self.dest, tag=self.tag, rm=True, decode=True) for entry in result: if "stream" in entry: sys.stdout.write(entry["stream"]) if "aux" in entry and "ID" in entry["aux"]: identity = entry["aux"]["ID"] except: logging.exception("Failed to create container.", exc_info=True) logging.warning( "You can manually create the container as follows:") logging.warning("docker build {}".format(self.dest)) self.identity = identity return identity def launch(self, image_sha, port=5555): """Launches the container with the given sha, publishing abd on port, and grpc on port + 1. Returns the container. """ client = docker.from_env() try: container = client.containers.run( image=image_sha, privileged=True, publish_all_ports=True, detach=True, ports={ "5555/tcp": port, "5556/tcp": port + 1 }, environment={"ADBKEY": self._read_adb_key()}, ) self.container = container print("Launched {} (id:{})".format(container.name, container.id)) print("docker logs -f {}".format(container.name)) print("docker stop {}".format(container.name)) return container except: logging.exception("Unable to run the %s", image_sha) print("Unable to start the container, try running it as:") print("./run.sh {}", image_sha) def create_docker_file(self, extra="", metrics=False): logging.info("Emulator zip: %s", self.emulator) logging.info("Sysimg zip: %s", self.sysimg) logging.info("Docker src dir: %s", self.dest) date = datetime.datetime.utcnow().isoformat("T") + "Z" # Make sure the destination directory is empty. if os.path.exists(self.dest): shutil.rmtree(self.dest) mkdir_p(self.dest) logging.info("Unzipping zips to docker src dir: %s", self.dest) extract_zip(self.emulator.fname, os.path.join(self.dest, 'emu')) extract_zip(self.sysimg.fname, os.path.join(self.dest, 'sys')) logging.info("Done unzipping") self._copy_adb_to(self.dest) self._write_template("avd/Pixel2.ini", {"api": self.sysimg.api()}) self._write_template( "avd/Pixel2.avd/config.ini", { "playstore": self.sysimg.tag() == "google_apis_playstore", "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "tag": self.sysimg.tag(), }, ) # Only version 29.3.1 >= can collect metrics. if metrics and version.parse( self.emulator.revision()) >= version.parse("29.3.1"): extra += " -metrics-collection" extra += " {}".format(self.sysimg.logger_flags()) self._write_template("launch-emulator.sh", { "extra": extra, "version": emu.__version__ }) self._write_template("default.pa", {}) self._write_template( "Dockerfile", { "user": "******".format(os.environ.get("USER", "unknown"), socket.gethostname()), "tag": self.sysimg.tag(), "api": self.sysimg.api(), "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "gpu": self.sysimg.gpu(), "emu_zip": os.path.basename(self.emulator.fname), "emu_build_id": self.emulator.build_id(), "sysimg_zip": os.path.basename(self.sysimg.fname), "date": date, "from_base_img": self.base_img, }, )
class DockerDevice(object): """A Docker Device is capable of creating and launching docker images. In order to successfully create and launch a docker image you must either run this as root, or have enabled sudoless docker. """ def __init__(self, emulator, sysimg, dest_dir, tag=""): self.sysimg = AndroidReleaseZip(sysimg) self.emulator = AndroidReleaseZip(emulator) self.tag = tag self.dest = dest_dir self.env = Environment(loader=PackageLoader("emu", "templates")) def _find_adb(self): adb_loc = os.path.join(os.environ.get("ANDROID_SDK_ROOT", ""), "platform-tools", "adb") if os.path.exists(adb_loc) and os.access(adb_loc, os.X_OK): return adb_loc return find_executable("adb") def _read_adb_key(self): adb_path = os.path.expanduser("~/.android/adbkey") if os.path.exists(adb_path): with open(adb_path, "r") as adb: return adb.read() return "" def _write_template(self, tmpl_file, template_dict): """Loads the the given template, writing it out to the dest_dir. Note: the template will be written {dest_dir}/{tmpl_file}, directories will be created if the do not yet exist. """ dest = os.path.join(self.dest, tmpl_file) dest_dir = os.path.dirname(dest) mkdir_p(dest_dir) logging.info("Writing: %s with %s", dest, template_dict) template = self.env.get_template(tmpl_file) with open(dest, "w") as dfile: dfile.write(template.render(template_dict)) def create_container(self, tag=None): """Creates the docker container, returning the sha of the container, or None in case of failure.""" if not tag: tag = "emulator/{}-{}-{}:{}".format(self.sysimg.tag(), self.sysimg.api(), self.sysimg.abi(), self.emulator.revision()) identity = None print("Creating docker image: {}.. be patient this can take a while!". format(tag)) try: api_client = docker.APIClient() logging.info(api_client.version()) result = api_client.build(path=self.dest, tag=self.tag, rm=True, decode=True) for entry in result: if "stream" in entry: sys.stdout.write(entry["stream"]) if "aux" in entry and "ID" in entry["aux"]: identity = entry["aux"]["ID"] except: logging.exception("Failed to create container.") print("You can manually create the container as follows:") print("docker build {}".format(self.dest)) return identity def launch(self, image_sha, port=5555): """Launches the container with the given sha, publishing abd on port, and grpc on port + 1.""" client = docker.from_env() try: container = client.containers.run( image=image_sha, privileged=True, publish_all_ports=True, detach=True, ports={ "5555/tcp": port, "5556/tcp": port + 1 }, environment={"ADBKEY": self._read_adb_key()}, ) print("Launched {} (id:{})".format(container.name, container.id)) print("docker logs -f {}".format(container.name)) print("docker stop {}".format(container.name)) except: logging.exception("Unable to run the %s", image_sha) print("Unable to start the container, try running it as:") print("./run.sh {}", image_sha) def create_docker_file(self, extra=""): logging.info("Emulator zip: %s", self.emulator) logging.info("Sysimg zip: %s", self.sysimg) logging.info("Docker src dir: %s", self.dest) adb_loc = self._find_adb() if adb_loc is None: raise IOError( errno.ENOENT, "Unable to find ADB below $ANDROID_SDK_ROOT or on the path!") date = datetime.datetime.utcnow().isoformat("T") + "Z" # Make sure the destination directory is empty. if os.path.exists(self.dest): shutil.rmtree(self.dest) mkdir_p(self.dest) logging.info("Copying zips to docker src dir: %s", self.dest) shutil.copy2(self.emulator.fname, self.dest) shutil.copy2(self.sysimg.fname, self.dest) logging.info("Done copying") platform_tools_dir = os.path.join(self.dest, "platform-tools") mkdir_p(platform_tools_dir) logging.info("Using adb: %s", adb_loc) shutil.copy2(adb_loc, platform_tools_dir) self._write_template("avd/Pixel2.ini", {"api": self.sysimg.api()}) self._write_template( "avd/Pixel2.avd/config.ini", { "playstore": self.sysimg.tag() == "google_apis_playstore", "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "tag": self.sysimg.tag(), }, ) extra += " {}".format(self.sysimg.logger_flags()) self._write_template("launch-emulator.sh", { "extra": extra, "version": emu.__version__ }) self._write_template("default.pa", {}) self._write_template( "Dockerfile", { "user": "******".format(os.environ["USER"], socket.gethostname()), "tag": self.sysimg.tag(), "api": self.sysimg.api(), "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "emu_zip": os.path.basename(self.emulator.fname), "emu_build_id": self.emulator.revision(), "sysimg_zip": os.path.basename(self.sysimg.fname), "date": date, }, )