class TemplateTest(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp("unittest")
self.writer = TemplateWriter(self.tmpdir)
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_writer_writes_file(self):
self.writer.write_template("cloudbuild.README.MD", {})
self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "cloudbuild.README.MD")))
def test_renames_file(self):
self.writer.write_template("cloudbuild.README.MD", {}, "foo")
self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "foo")))
def write(self, dest):
# We do not really want to overwrite if the files already exist.
# Make sure the destination directory is empty.
if self.system_image_zip is None:
self.system_image_zip = SystemImageReleaseZip(
self.system_image_info.download(dest))
writer = TemplateWriter(dest)
self._copy_adb_to(dest)
props = self.system_image_zip.props
dest_zip = os.path.basename(self.system_image_zip.copy(dest))
props["system_image_zip"] = dest_zip
writer.write_template(
"Dockerfile.system_image",
props,
rename_as="Dockerfile",
)
def cloud_build(args):
licenses = set([x.license for x in emu_downloads_menu.get_emus_info()] +
[x.license for x in emu_downloads_menu.get_images_info()])
for l in licenses:
l.force_accept()
imgzip = [x.download() for x in emu_downloads_menu.find_image(args.img)]
emuzip = [x.download() for x in emu_downloads_menu.find_emulator("canary")]
devices = []
steps = []
images = []
emulators = set()
for (img, emu) in itertools.product(imgzip, emuzip):
logging.info("Processing %s, %s", img, emu)
img_rel = emu_downloads_menu.AndroidReleaseZip(img)
if not img_rel.is_system_image():
logging.warn(
"{} is not a zip file with a system image (Unexpected description), skipping"
.format(img))
continue
emu_rel = emu_downloads_menu.AndroidReleaseZip(emu)
if not emu_rel.is_emulator():
raise Exception(
"{} is not a zip file with an emulator".format(emu))
emulators.add(emu_rel.build_id())
for metrics in [True, False]:
name = img_rel.repo_friendly_name()
if not metrics:
name += "-no-metrics"
dest = os.path.join(args.dest, name)
logging.info("Generating %s", name)
device = DockerDevice(emu,
img,
dest,
gpu=False,
repo=args.repo,
tag=emu_rel.build_id(),
name=name)
device.create_docker_file("",
metrics=True,
by_copying_zip_files=True)
steps.append(device.create_cloud_build_step())
images.append(device.tag)
cloudbuild = {"steps": steps, "images": images}
with open(os.path.join(args.dest, "cloudbuild.yaml"), "w") as ymlfile:
yaml.dump(cloudbuild, ymlfile)
writer = TemplateWriter(args.dest)
writer.write_template(
"cloudbuild.README.MD",
{
"emu_version": ", ".join(emulators),
"emu_images": "\n".join(images)
},
rename_as="README.MD",
)
def cloud_build(args):
"""Prepares the cloud build yaml and all its dependencies.
The cloud builder will generate a single cloudbuild.yaml and generates the build
scripts for every individual container.
It will construct the proper dependencies as needed.
"""
accept_licenses(True)
mkdir_p(args.dest)
image_zip = [args.img]
# Check if we are building a custom image from a zip file
if not os.path.exists(image_zip[0]):
# We are using a standard image, we likely won't need to download it.
image_zip = emu_downloads_menu.find_image(image_zip[0])
emulator_zip = [args.emuzip]
if emulator_zip[0] in ["stable", "canary", "all"]:
emulator_zip = [
x.download()
for x in emu_downloads_menu.find_emulator(emulator_zip[0])
]
elif re.match(r"\d+", emulator_zip[0]):
# We must be looking for a build id
logging.warning("Treating %s as a build id", emulator_zip[0])
emulator_zip = [emu_downloads_menu.download_build(emulator_zip[0])]
steps = []
images = []
emulators = set()
emulator_images = []
for (img, emu) in itertools.product(image_zip, emulator_zip):
logging.info("Processing %s, %s", img, emu)
system_container = SystemImageContainer(img, args.repo)
if args.sys:
steps.append(create_build_step(system_container, args.dest))
else:
for metrics in [True, False]:
emulator_container = EmulatorContainer(emu, system_container,
args.repo, metrics)
emulators.add(emulator_container.props["emu_build_id"])
steps.append(create_build_step(emulator_container, args.dest))
images.append(emulator_container.full_name())
images.append(emulator_container.latest_name())
emulator_images.append(emulator_container.full_name())
emulator_images.append(emulator_container.latest_name())
cloudbuild = {"steps": steps, "images": images, "timeout": "21600s"}
logging.info("Writing cloud yaml [%s] in %s", yaml, args.dest)
with open(os.path.join(args.dest, "cloudbuild.yaml"), "w") as ymlfile:
yaml.dump(cloudbuild, ymlfile)
writer = TemplateWriter(args.dest)
writer.write_template(
"cloudbuild.README.MD",
{
"emu_version": ", ".join(emulators),
"emu_images": "\n".join(
["* {}".format(x) for x in emulator_images])
},
rename_as="README.MD",
)
writer.write_template(
"registry.README.MD",
{
"emu_version": ", ".join(emulators),
"emu_images": "\n".join(["* {}".format(x) for x in images]),
"first_image": next(iter(images), None),
},
rename_as="REGISTRY.MD",
)
if args.git:
git_commit_and_push(args.dest)
class DockerDevice(object):
"""A Docker Device is capable of creating and launching docker images.
In order to successfully create and launch a docker image you must either
run this as root, or have enabled sudoless docker.
"""
TAG_REGEX = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]*:?[a-zA-Z0-9._-]*")
GPU_BASEIMG = (
"FROM nvidia/opengl:1.0-glvnd-runtime-ubuntu18.04 AS emulator\n"
+ "ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display"
)
DEFAULT_BASE_IMG = "FROM debian:stretch-slim AS emulator"
def __init__(self, emulator, sysimg, dest_dir, gpu=False, repo="", tag="", name=""):
self.sysimg = AndroidReleaseZip(sysimg)
self.emulator = AndroidReleaseZip(emulator)
self.dest = dest_dir
self.writer = TemplateWriter(dest_dir)
if repo and repo[-1] != "/":
repo += "/"
if not name:
name = self.sysimg.repo_friendly_name()
repo += name
if gpu:
repo += "-gpu"
if not tag:
tag = self.emulator.build_id()
self.tag = "{}:{}".format(repo, tag)
if not self.TAG_REGEX.match(self.tag):
raise Exception("The resulting tag: {} is not a valid docker tag.", self.tag)
# The following are only set after creating/launching.
self.container = None
self.identity = None
self.base_img = DockerDevice.DEFAULT_BASE_IMG
if gpu:
self.base_img = DockerDevice.GPU_BASEIMG
def push(self, sha, latest=False):
repo, tag = self.tag.split(":")
print("Pushing docker image: {}.. be patient this can take a while!".format(self.tag))
tracker = ProgressTracker()
try:
client = docker.from_env()
result = client.images.push(repo, tag, stream=True, decode=True)
for entry in result:
tracker.update(entry)
except:
logging.exception("Failed to push image.", exc_info=True)
logging.warning("You can manually push the image as follows:")
logging.warning("docker push {}".format(self.tag))
def _copy_adb_to(self, dest):
"""Find adb, or download it if needed."""
logging.info("Retrieving platform-tools")
tools = PlatformTools()
tools.extract_adb(dest)
def _read_adb_key(self):
adb_path = os.path.expanduser("~/.android/adbkey")
if os.path.exists(adb_path):
with open(adb_path, "r") as adb:
return adb.read()
return ""
def get_api_client(self):
try:
api_client = docker.APIClient()
logging.info(api_client.version())
return api_client
except:
logging.exception("Failed to create default client, trying domain socket.", exc_info=True)
api_client = docker.APIClient(base_url="unix://var/run/docker.sock")
logging.info(api_client.version())
return api_client
def create_container(self):
"""Creates the docker container, returning the sha of the container, or None in case of failure."""
identity = None
print("Creating docker image: {}.. be patient this can take a while!".format(self.tag))
try:
logging.info("build(path=%s, tag=%s, rm=True, decode=True)", self.dest, self.tag)
api_client = self.get_api_client()
result = api_client.build(path=self.dest, tag=self.tag, rm=True, decode=True)
for entry in result:
if "stream" in entry:
sys.stdout.write(entry["stream"])
if "aux" in entry and "ID" in entry["aux"]:
identity = entry["aux"]["ID"]
except:
logging.exception("Failed to create container.", exc_info=True)
logging.warning("You can manually create the container as follows:")
logging.warning("docker build {}".format(self.dest))
self.identity = identity
return identity
def create_cloud_build_step(self):
return {"name": "gcr.io/cloud-builders/docker", "args": ["build", "-t", self.tag, os.path.basename(self.dest)]}
def launch(self, image_sha, port=5555):
"""Launches the container with the given sha, publishing abd on port, and grpc on port 8554
Returns the container.
"""
client = docker.from_env()
try:
container = client.containers.run(
image=image_sha,
privileged=True,
publish_all_ports=True,
detach=True,
ports={"5555/tcp": port, "8554/tcp": 8554},
environment={"ADBKEY": self._read_adb_key()},
)
self.container = container
print("Launched {} (id:{})".format(container.name, container.id))
print("docker logs -f {}".format(container.name))
print("docker stop {}".format(container.name))
return container
except:
logging.exception("Unable to run the %s", image_sha)
print("Unable to start the container, try running it as:")
print("./run.sh {}", image_sha)
def bin_place_emulator_files(self, by_copying_zip_files):
"""Bin places the emulator files for the docker file."""
if by_copying_zip_files:
logging.info("Copying zips to docker src dir: %s", self.dest)
shutil.copy2(self.emulator.fname, self.dest)
shutil.copy2(self.sysimg.fname, self.dest)
logging.info("Done copying")
else:
logging.info("Unzipping zips to docker src dir: %s", self.dest)
extract_zip(self.emulator.fname, os.path.join(self.dest, "emu"))
extract_zip(self.sysimg.fname, os.path.join(self.dest, "sys"))
logging.info("Done unzipping")
def create_docker_file(self, extra="", metrics=False, by_copying_zip_files=False):
logging.info("Emulator zip: %s", self.emulator)
logging.info("Sysimg zip: %s", self.sysimg)
logging.info("Docker src dir: %s", self.dest)
date = datetime.datetime.utcnow().isoformat("T") + "Z"
# Make sure the destination directory is empty.
if os.path.exists(self.dest):
shutil.rmtree(self.dest)
mkdir_p(self.dest)
self.bin_place_emulator_files(by_copying_zip_files)
self._copy_adb_to(self.dest)
self.writer.write_template("avd/Pixel2.ini", {"api": self.sysimg.api()})
self.writer.write_template(
"avd/Pixel2.avd/config.ini",
{
"playstore": self.sysimg.tag() == "google_apis_playstore",
"abi": self.sysimg.abi(),
"cpu": self.sysimg.cpu(),
"tag": self.sysimg.tag(),
},
)
# Only version 29.3.1 >= can collect metrics.
if metrics and version.parse(self.emulator.revision()) >= version.parse("29.3.1"):
extra += " -metrics-collection"
extra += " {}".format(self.sysimg.logger_flags())
self.writer.write_template("launch-emulator.sh", {"extra": extra, "version": emu.__version__})
self.writer.write_template("default.pa", {})
src_template = "Dockerfile.from_zip" if by_copying_zip_files else "Dockerfile"
self.writer.write_template(
src_template,
{
"user": "******".format(os.environ.get("USER", "unknown"), socket.gethostname()),
"tag": self.sysimg.tag(),
"api": self.sysimg.api(),
"abi": self.sysimg.abi(),
"cpu": self.sysimg.cpu(),
"gpu": self.sysimg.gpu(),
"emu_zip": os.path.basename(self.emulator.fname),
"emu_build_id": self.emulator.build_id(),
"sysimg_zip": os.path.basename(self.sysimg.fname),
"date": date,
"from_base_img": self.base_img,
},
rename_as="Dockerfile",
)
def write(self, dest):
# Make sure the destination directory is empty.
self.clean(dest)
writer = TemplateWriter(dest)
writer.write_template("avd/Pixel2.ini", self.props)
writer.write_template("avd/Pixel2.avd/config.ini", self.props)
# Include a README.MD message.
writer.write_template(
"emulator.README.MD",
self.props,
rename_as="README.MD",
)
writer.write_template("launch-emulator.sh", {
"extra": self.extra,
"version": emu.__version__
})
writer.write_template("default.pa", {})
writer.write_template(
"Dockerfile.emulator",
self.props,
rename_as="Dockerfile",
)
self.emulator_zip.extract(os.path.join(dest, "emu"))
def cloud_build(args):
licenses = set(
[x.license for x in emu_downloads_menu.get_emus_info()]
+ [x.license for x in emu_downloads_menu.get_images_info()]
)
for l in licenses:
l.force_accept()
imgzip = [x.download() for x in emu_downloads_menu.find_image(args.img)]
emuzip = [args.emuzip]
if emuzip[0] in ["stable", "canary", "all"]:
emuzip = [x.download() for x in emu_downloads_menu.find_emulator(emuzip[0])]
elif re.match("\d+", emuzip[0]):
# We must be looking for a build id
logging.info("Treating %s as a build id", emuzip[0])
emuzip = [emu_downloads_menu.download_build(emuzip[0])]
steps = []
images = []
desserts = set()
emulators = set()
for (img, emu) in itertools.product(imgzip, emuzip):
logging.info("Processing %s, %s", img, emu)
img_rel = emu_downloads_menu.AndroidReleaseZip(img)
if not img_rel.is_system_image():
logging.warning("{} is not a zip file with a system image (Unexpected description), skipping".format(img))
continue
emu_rel = emu_downloads_menu.AndroidReleaseZip(emu)
if not emu_rel.is_emulator():
raise Exception("{} is not a zip file with an emulator".format(emu))
emulators.add(emu_rel.build_id())
desserts.add(img_rel.codename())
for metrics in [True, False]:
name = img_rel.repo_friendly_name()
if not metrics:
name += "-no-metrics"
dest = os.path.join(args.dest, name)
logging.info("Generating %s", name)
device = DockerDevice(emu, img, dest, gpu=False, repo=args.repo, tag=emu_rel.build_id(), name=name)
device.create_docker_file("", metrics=True, by_copying_zip_files=True)
steps.append(device.create_cloud_build_step())
images.append(device.tag)
images.append(device.latest)
steps[-1]["waitFor"] = ["-"]
cloudbuild = {"steps": steps, "images": images, "timeout": "21600s"}
with open(os.path.join(args.dest, "cloudbuild.yaml"), "w") as ymlfile:
yaml.dump(cloudbuild, ymlfile)
writer = TemplateWriter(args.dest)
writer.write_template(
"cloudbuild.README.MD",
{"emu_version": ", ".join(emulators), "emu_images": "\n".join(["* {}".format(x) for x in images])},
rename_as="README.MD",
)
writer.write_template(
"registry.README.MD",
{
"emu_version": ", ".join(emulators),
"emu_images": "\n".join(["* {}".format(x) for x in images]),
"first_image": images[0]
},
rename_as="REGISTRY.MD",
)
if args.git:
git_commit_and_push(args.dest)