class TemplateTest(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp("unittest") self.writer = TemplateWriter(self.tmpdir) def tearDown(self): shutil.rmtree(self.tmpdir) def test_writer_writes_file(self): self.writer.write_template("cloudbuild.README.MD", {}) self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "cloudbuild.README.MD"))) def test_renames_file(self): self.writer.write_template("cloudbuild.README.MD", {}, "foo") self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "foo")))
def write(self, dest): # We do not really want to overwrite if the files already exist. # Make sure the destination directory is empty. if self.system_image_zip is None: self.system_image_zip = SystemImageReleaseZip( self.system_image_info.download(dest)) writer = TemplateWriter(dest) self._copy_adb_to(dest) props = self.system_image_zip.props dest_zip = os.path.basename(self.system_image_zip.copy(dest)) props["system_image_zip"] = dest_zip writer.write_template( "Dockerfile.system_image", props, rename_as="Dockerfile", )
def cloud_build(args): licenses = set([x.license for x in emu_downloads_menu.get_emus_info()] + [x.license for x in emu_downloads_menu.get_images_info()]) for l in licenses: l.force_accept() imgzip = [x.download() for x in emu_downloads_menu.find_image(args.img)] emuzip = [x.download() for x in emu_downloads_menu.find_emulator("canary")] devices = [] steps = [] images = [] emulators = set() for (img, emu) in itertools.product(imgzip, emuzip): logging.info("Processing %s, %s", img, emu) img_rel = emu_downloads_menu.AndroidReleaseZip(img) if not img_rel.is_system_image(): logging.warn( "{} is not a zip file with a system image (Unexpected description), skipping" .format(img)) continue emu_rel = emu_downloads_menu.AndroidReleaseZip(emu) if not emu_rel.is_emulator(): raise Exception( "{} is not a zip file with an emulator".format(emu)) emulators.add(emu_rel.build_id()) for metrics in [True, False]: name = img_rel.repo_friendly_name() if not metrics: name += "-no-metrics" dest = os.path.join(args.dest, name) logging.info("Generating %s", name) device = DockerDevice(emu, img, dest, gpu=False, repo=args.repo, tag=emu_rel.build_id(), name=name) device.create_docker_file("", metrics=True, by_copying_zip_files=True) steps.append(device.create_cloud_build_step()) images.append(device.tag) cloudbuild = {"steps": steps, "images": images} with open(os.path.join(args.dest, "cloudbuild.yaml"), "w") as ymlfile: yaml.dump(cloudbuild, ymlfile) writer = TemplateWriter(args.dest) writer.write_template( "cloudbuild.README.MD", { "emu_version": ", ".join(emulators), "emu_images": "\n".join(images) }, rename_as="README.MD", )
def cloud_build(args): """Prepares the cloud build yaml and all its dependencies. The cloud builder will generate a single cloudbuild.yaml and generates the build scripts for every individual container. It will construct the proper dependencies as needed. """ accept_licenses(True) mkdir_p(args.dest) image_zip = [args.img] # Check if we are building a custom image from a zip file if not os.path.exists(image_zip[0]): # We are using a standard image, we likely won't need to download it. image_zip = emu_downloads_menu.find_image(image_zip[0]) emulator_zip = [args.emuzip] if emulator_zip[0] in ["stable", "canary", "all"]: emulator_zip = [ x.download() for x in emu_downloads_menu.find_emulator(emulator_zip[0]) ] elif re.match(r"\d+", emulator_zip[0]): # We must be looking for a build id logging.warning("Treating %s as a build id", emulator_zip[0]) emulator_zip = [emu_downloads_menu.download_build(emulator_zip[0])] steps = [] images = [] emulators = set() emulator_images = [] for (img, emu) in itertools.product(image_zip, emulator_zip): logging.info("Processing %s, %s", img, emu) system_container = SystemImageContainer(img, args.repo) if args.sys: steps.append(create_build_step(system_container, args.dest)) else: for metrics in [True, False]: emulator_container = EmulatorContainer(emu, system_container, args.repo, metrics) emulators.add(emulator_container.props["emu_build_id"]) steps.append(create_build_step(emulator_container, args.dest)) images.append(emulator_container.full_name()) images.append(emulator_container.latest_name()) emulator_images.append(emulator_container.full_name()) emulator_images.append(emulator_container.latest_name()) cloudbuild = {"steps": steps, "images": images, "timeout": "21600s"} logging.info("Writing cloud yaml [%s] in %s", yaml, args.dest) with open(os.path.join(args.dest, "cloudbuild.yaml"), "w") as ymlfile: yaml.dump(cloudbuild, ymlfile) writer = TemplateWriter(args.dest) writer.write_template( "cloudbuild.README.MD", { "emu_version": ", ".join(emulators), "emu_images": "\n".join( ["* {}".format(x) for x in emulator_images]) }, rename_as="README.MD", ) writer.write_template( "registry.README.MD", { "emu_version": ", ".join(emulators), "emu_images": "\n".join(["* {}".format(x) for x in images]), "first_image": next(iter(images), None), }, rename_as="REGISTRY.MD", ) if args.git: git_commit_and_push(args.dest)
class DockerDevice(object): """A Docker Device is capable of creating and launching docker images. In order to successfully create and launch a docker image you must either run this as root, or have enabled sudoless docker. """ TAG_REGEX = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]*:?[a-zA-Z0-9._-]*") GPU_BASEIMG = ( "FROM nvidia/opengl:1.0-glvnd-runtime-ubuntu18.04 AS emulator\n" + "ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display" ) DEFAULT_BASE_IMG = "FROM debian:stretch-slim AS emulator" def __init__(self, emulator, sysimg, dest_dir, gpu=False, repo="", tag="", name=""): self.sysimg = AndroidReleaseZip(sysimg) self.emulator = AndroidReleaseZip(emulator) self.dest = dest_dir self.writer = TemplateWriter(dest_dir) if repo and repo[-1] != "/": repo += "/" if not name: name = self.sysimg.repo_friendly_name() repo += name if gpu: repo += "-gpu" if not tag: tag = self.emulator.build_id() self.tag = "{}:{}".format(repo, tag) if not self.TAG_REGEX.match(self.tag): raise Exception("The resulting tag: {} is not a valid docker tag.", self.tag) # The following are only set after creating/launching. self.container = None self.identity = None self.base_img = DockerDevice.DEFAULT_BASE_IMG if gpu: self.base_img = DockerDevice.GPU_BASEIMG def push(self, sha, latest=False): repo, tag = self.tag.split(":") print("Pushing docker image: {}.. be patient this can take a while!".format(self.tag)) tracker = ProgressTracker() try: client = docker.from_env() result = client.images.push(repo, tag, stream=True, decode=True) for entry in result: tracker.update(entry) except: logging.exception("Failed to push image.", exc_info=True) logging.warning("You can manually push the image as follows:") logging.warning("docker push {}".format(self.tag)) def _copy_adb_to(self, dest): """Find adb, or download it if needed.""" logging.info("Retrieving platform-tools") tools = PlatformTools() tools.extract_adb(dest) def _read_adb_key(self): adb_path = os.path.expanduser("~/.android/adbkey") if os.path.exists(adb_path): with open(adb_path, "r") as adb: return adb.read() return "" def get_api_client(self): try: api_client = docker.APIClient() logging.info(api_client.version()) return api_client except: logging.exception("Failed to create default client, trying domain socket.", exc_info=True) api_client = docker.APIClient(base_url="unix://var/run/docker.sock") logging.info(api_client.version()) return api_client def create_container(self): """Creates the docker container, returning the sha of the container, or None in case of failure.""" identity = None print("Creating docker image: {}.. be patient this can take a while!".format(self.tag)) try: logging.info("build(path=%s, tag=%s, rm=True, decode=True)", self.dest, self.tag) api_client = self.get_api_client() result = api_client.build(path=self.dest, tag=self.tag, rm=True, decode=True) for entry in result: if "stream" in entry: sys.stdout.write(entry["stream"]) if "aux" in entry and "ID" in entry["aux"]: identity = entry["aux"]["ID"] except: logging.exception("Failed to create container.", exc_info=True) logging.warning("You can manually create the container as follows:") logging.warning("docker build {}".format(self.dest)) self.identity = identity return identity def create_cloud_build_step(self): return {"name": "gcr.io/cloud-builders/docker", "args": ["build", "-t", self.tag, os.path.basename(self.dest)]} def launch(self, image_sha, port=5555): """Launches the container with the given sha, publishing abd on port, and grpc on port 8554 Returns the container. """ client = docker.from_env() try: container = client.containers.run( image=image_sha, privileged=True, publish_all_ports=True, detach=True, ports={"5555/tcp": port, "8554/tcp": 8554}, environment={"ADBKEY": self._read_adb_key()}, ) self.container = container print("Launched {} (id:{})".format(container.name, container.id)) print("docker logs -f {}".format(container.name)) print("docker stop {}".format(container.name)) return container except: logging.exception("Unable to run the %s", image_sha) print("Unable to start the container, try running it as:") print("./run.sh {}", image_sha) def bin_place_emulator_files(self, by_copying_zip_files): """Bin places the emulator files for the docker file.""" if by_copying_zip_files: logging.info("Copying zips to docker src dir: %s", self.dest) shutil.copy2(self.emulator.fname, self.dest) shutil.copy2(self.sysimg.fname, self.dest) logging.info("Done copying") else: logging.info("Unzipping zips to docker src dir: %s", self.dest) extract_zip(self.emulator.fname, os.path.join(self.dest, "emu")) extract_zip(self.sysimg.fname, os.path.join(self.dest, "sys")) logging.info("Done unzipping") def create_docker_file(self, extra="", metrics=False, by_copying_zip_files=False): logging.info("Emulator zip: %s", self.emulator) logging.info("Sysimg zip: %s", self.sysimg) logging.info("Docker src dir: %s", self.dest) date = datetime.datetime.utcnow().isoformat("T") + "Z" # Make sure the destination directory is empty. if os.path.exists(self.dest): shutil.rmtree(self.dest) mkdir_p(self.dest) self.bin_place_emulator_files(by_copying_zip_files) self._copy_adb_to(self.dest) self.writer.write_template("avd/Pixel2.ini", {"api": self.sysimg.api()}) self.writer.write_template( "avd/Pixel2.avd/config.ini", { "playstore": self.sysimg.tag() == "google_apis_playstore", "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "tag": self.sysimg.tag(), }, ) # Only version 29.3.1 >= can collect metrics. if metrics and version.parse(self.emulator.revision()) >= version.parse("29.3.1"): extra += " -metrics-collection" extra += " {}".format(self.sysimg.logger_flags()) self.writer.write_template("launch-emulator.sh", {"extra": extra, "version": emu.__version__}) self.writer.write_template("default.pa", {}) src_template = "Dockerfile.from_zip" if by_copying_zip_files else "Dockerfile" self.writer.write_template( src_template, { "user": "******".format(os.environ.get("USER", "unknown"), socket.gethostname()), "tag": self.sysimg.tag(), "api": self.sysimg.api(), "abi": self.sysimg.abi(), "cpu": self.sysimg.cpu(), "gpu": self.sysimg.gpu(), "emu_zip": os.path.basename(self.emulator.fname), "emu_build_id": self.emulator.build_id(), "sysimg_zip": os.path.basename(self.sysimg.fname), "date": date, "from_base_img": self.base_img, }, rename_as="Dockerfile", )
def write(self, dest): # Make sure the destination directory is empty. self.clean(dest) writer = TemplateWriter(dest) writer.write_template("avd/Pixel2.ini", self.props) writer.write_template("avd/Pixel2.avd/config.ini", self.props) # Include a README.MD message. writer.write_template( "emulator.README.MD", self.props, rename_as="README.MD", ) writer.write_template("launch-emulator.sh", { "extra": self.extra, "version": emu.__version__ }) writer.write_template("default.pa", {}) writer.write_template( "Dockerfile.emulator", self.props, rename_as="Dockerfile", ) self.emulator_zip.extract(os.path.join(dest, "emu"))
def cloud_build(args): licenses = set( [x.license for x in emu_downloads_menu.get_emus_info()] + [x.license for x in emu_downloads_menu.get_images_info()] ) for l in licenses: l.force_accept() imgzip = [x.download() for x in emu_downloads_menu.find_image(args.img)] emuzip = [args.emuzip] if emuzip[0] in ["stable", "canary", "all"]: emuzip = [x.download() for x in emu_downloads_menu.find_emulator(emuzip[0])] elif re.match("\d+", emuzip[0]): # We must be looking for a build id logging.info("Treating %s as a build id", emuzip[0]) emuzip = [emu_downloads_menu.download_build(emuzip[0])] steps = [] images = [] desserts = set() emulators = set() for (img, emu) in itertools.product(imgzip, emuzip): logging.info("Processing %s, %s", img, emu) img_rel = emu_downloads_menu.AndroidReleaseZip(img) if not img_rel.is_system_image(): logging.warning("{} is not a zip file with a system image (Unexpected description), skipping".format(img)) continue emu_rel = emu_downloads_menu.AndroidReleaseZip(emu) if not emu_rel.is_emulator(): raise Exception("{} is not a zip file with an emulator".format(emu)) emulators.add(emu_rel.build_id()) desserts.add(img_rel.codename()) for metrics in [True, False]: name = img_rel.repo_friendly_name() if not metrics: name += "-no-metrics" dest = os.path.join(args.dest, name) logging.info("Generating %s", name) device = DockerDevice(emu, img, dest, gpu=False, repo=args.repo, tag=emu_rel.build_id(), name=name) device.create_docker_file("", metrics=True, by_copying_zip_files=True) steps.append(device.create_cloud_build_step()) images.append(device.tag) images.append(device.latest) steps[-1]["waitFor"] = ["-"] cloudbuild = {"steps": steps, "images": images, "timeout": "21600s"} with open(os.path.join(args.dest, "cloudbuild.yaml"), "w") as ymlfile: yaml.dump(cloudbuild, ymlfile) writer = TemplateWriter(args.dest) writer.write_template( "cloudbuild.README.MD", {"emu_version": ", ".join(emulators), "emu_images": "\n".join(["* {}".format(x) for x in images])}, rename_as="README.MD", ) writer.write_template( "registry.README.MD", { "emu_version": ", ".join(emulators), "emu_images": "\n".join(["* {}".format(x) for x in images]), "first_image": images[0] }, rename_as="REGISTRY.MD", ) if args.git: git_commit_and_push(args.dest)