def test_create_context_missing_extra(self):
tmp = tempfile.mkdtemp()
try:
d = os.path.join(tmp, "test-image")
os.mkdir(d)
with open(os.path.join(d, "Dockerfile"), "wb") as fh:
fh.write(b"# %include does/not/exist\n")
with self.assertRaisesRegexp(Exception, "path does not exist"):
docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"),
"test", {})
finally:
shutil.rmtree(tmp)
def test_create_context_outside_topsrcdir(self):
tmp = tempfile.mkdtemp()
try:
d = os.path.join(tmp, "test-image")
os.mkdir(d)
with open(os.path.join(d, "Dockerfile"), "wb") as fh:
fh.write(b"# %include foo/../../../etc/shadow\n")
with self.assertRaisesRegexp(Exception, "path outside topsrcdir"):
docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"),
"test", {})
finally:
shutil.rmtree(tmp)
def test_create_context_absolute_path(self):
tmp = tempfile.mkdtemp()
try:
d = os.path.join(tmp, "test-image")
os.mkdir(d)
# Absolute paths in %include syntax are not allowed.
with open(os.path.join(d, "Dockerfile"), "wb") as fh:
fh.write(b"# %include /etc/shadow\n")
with self.assertRaisesRegexp(Exception, "cannot be absolute"):
docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"),
"test", {})
finally:
shutil.rmtree(tmp)
def build_context(name, outputFile, args=None):
"""Build a context.tar for image with specified name."""
if not name:
raise ValueError("must provide a Docker image name")
if not outputFile:
raise ValueError("must provide a outputFile")
image_dir = docker.image_path(name)
if not os.path.isdir(image_dir):
raise Exception("image directory does not exist: %s" % image_dir)
docker.create_context_tar(GECKO,
image_dir,
outputFile,
image_name=name,
args=args)
def test_create_context_tar_basic(self):
tmp = tempfile.mkdtemp()
try:
d = os.path.join(tmp, "test_image")
os.mkdir(d)
with open(os.path.join(d, "Dockerfile"), "a"):
pass
os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD)
with open(os.path.join(d, "extra"), "a"):
pass
os.chmod(os.path.join(d, "extra"), MODE_STANDARD)
tp = os.path.join(tmp, "tar")
h = docker.create_context_tar(tmp, d, tp, "my_image", {})
self.assertEqual(
h,
"eae3ad00936085eb3e5958912f79fb06ee8e14a91f7157c5f38625f7ddacb9c7"
)
# File prefix should be "my_image"
with tarfile.open(tp, "r:gz") as tf:
self.assertEqual(
tf.getnames(),
[
"Dockerfile",
"extra",
],
)
finally:
shutil.rmtree(tmp)
def test_create_context_topsrcdir_files(self):
tmp = tempfile.mkdtemp()
try:
d = os.path.join(tmp, "test-image")
os.mkdir(d)
with open(os.path.join(d, "Dockerfile"), "wb") as fh:
fh.write(b"# %include extra/file0\n")
os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD)
extra = os.path.join(tmp, "extra")
os.mkdir(extra)
with open(os.path.join(extra, "file0"), "a"):
pass
os.chmod(os.path.join(extra, "file0"), MODE_STANDARD)
tp = os.path.join(tmp, "tar")
h = docker.create_context_tar(tmp, d, tp, "test_image", {})
self.assertEqual(
h,
"49dc3827530cd344d7bcc52e1fdd4aefc632568cf442cffd3dd9633a58f271bf"
)
with tarfile.open(tp, "r:gz") as tf:
self.assertEqual(
tf.getnames(),
[
"Dockerfile",
"topsrcdir/extra/file0",
],
)
finally:
shutil.rmtree(tmp)
def test_create_context_extra_directory(self):
tmp = tempfile.mkdtemp()
try:
d = os.path.join(tmp, "test-image")
os.mkdir(d)
with open(os.path.join(d, "Dockerfile"), "wb") as fh:
fh.write(b"# %include extra\n")
fh.write(b"# %include file0\n")
os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD)
extra = os.path.join(tmp, "extra")
os.mkdir(extra)
for i in range(3):
p = os.path.join(extra, "file%d" % i)
with open(p, "wb") as fh:
fh.write(b"file%d" % i)
os.chmod(p, MODE_STANDARD)
with open(os.path.join(tmp, "file0"), "a"):
pass
os.chmod(os.path.join(tmp, "file0"), MODE_STANDARD)
tp = os.path.join(tmp, "tar")
h = docker.create_context_tar(tmp, d, tp, "my_image", {})
self.assertEqual(
h,
"a392f23cd6606ae43116390a4d0113354cff1e688a41d46f48b0fb25e90baa13"
)
with tarfile.open(tp, "r:gz") as tf:
self.assertEqual(
tf.getnames(),
[
"Dockerfile",
"topsrcdir/extra/file0",
"topsrcdir/extra/file1",
"topsrcdir/extra/file2",
"topsrcdir/file0",
],
)
finally:
shutil.rmtree(tmp)
def fill_template(config, tasks):
if not gecko_taskgraph.fast and config.write_artifacts:
if not os.path.isdir(CONTEXTS_DIR):
os.makedirs(CONTEXTS_DIR)
for task in tasks:
image_name = task.pop("name")
job_symbol = task.pop("symbol")
args = task.pop("args", {})
packages = task.pop("packages", [])
parent = task.pop("parent", None)
for p in packages:
if f"packages-{p}" not in config.kind_dependencies_tasks:
raise Exception("Missing package job for {}-{}: {}".format(
config.kind, image_name, p))
if not gecko_taskgraph.fast:
context_path = mozpath.relpath(image_path(image_name), GECKO)
if config.write_artifacts:
context_file = os.path.join(CONTEXTS_DIR,
f"{image_name}.tar.gz")
logger.info(
f"Writing {context_file} for docker image {image_name}")
context_hash = create_context_tar(GECKO, context_path,
context_file, image_name,
args)
else:
context_hash = generate_context_hash(GECKO, context_path,
image_name, args)
else:
if config.write_artifacts:
raise Exception(
"Can't write artifacts if `gecko_taskgraph.fast` is set.")
context_hash = "0" * 40
digest_data = [context_hash]
digest_data += [json.dumps(args, sort_keys=True)]
description = "Build the docker image {} for use by dependent tasks".format(
image_name)
args["DOCKER_IMAGE_PACKAGES"] = " ".join(f"<{p}>" for p in packages)
# Adjust the zstandard compression level based on the execution level.
# We use faster compression for level 1 because we care more about
# end-to-end times. We use slower/better compression for other levels
# because images are read more often and it is worth the trade-off to
# burn more CPU once to reduce image size.
zstd_level = "3" if int(config.params["level"]) == 1 else "10"
# include some information that is useful in reconstructing this task
# from JSON
taskdesc = {
"label": f"{config.kind}-{image_name}",
"description": description,
"attributes": {
"image_name": image_name,
"artifact_prefix": "public",
},
"expires-after": "1 year",
"scopes": [],
"treeherder": {
"symbol": job_symbol,
"platform": "taskcluster-images/opt",
"kind": "other",
"tier": 1,
},
"run-on-projects": [],
"worker-type": "images",
"worker": {
"implementation":
"docker-worker",
"os":
"linux",
"artifacts": [{
"type": "file",
"path": "/workspace/image.tar.zst",
"name": "public/image.tar.zst",
}],
"env": {
"CONTEXT_TASK_ID": {
"task-reference": "<decision>"
},
"CONTEXT_PATH":
"public/docker-contexts/{}.tar.gz".format(image_name),
"HASH":
context_hash,
"PROJECT":
config.params["project"],
"IMAGE_NAME":
image_name,
"DOCKER_IMAGE_ZSTD_LEVEL":
zstd_level,
"DOCKER_BUILD_ARGS": {
"task-reference": json.dumps(args)
},
"GECKO_BASE_REPOSITORY":
config.params["base_repository"],
"GECKO_HEAD_REPOSITORY":
config.params["head_repository"],
"GECKO_HEAD_REV":
config.params["head_rev"],
},
"chain-of-trust":
True,
"max-run-time":
7200,
# FIXME: We aren't currently propagating the exit code
},
}
# Retry for 'funsize-update-generator' if exit status code is -1
if image_name in ["funsize-update-generator"]:
taskdesc["worker"]["retry-exit-status"] = [-1]
worker = taskdesc["worker"]
if image_name == "image_builder":
worker["docker-image"] = IMAGE_BUILDER_IMAGE
digest_data.append(f"image-builder-image:{IMAGE_BUILDER_IMAGE}")
else:
worker["docker-image"] = {"in-tree": "image_builder"}
deps = taskdesc.setdefault("dependencies", {})
deps["docker-image"] = f"{config.kind}-image_builder"
if packages:
deps = taskdesc.setdefault("dependencies", {})
for p in sorted(packages):
deps[p] = f"packages-{p}"
if parent:
deps = taskdesc.setdefault("dependencies", {})
deps["parent"] = f"{config.kind}-{parent}"
worker["env"]["PARENT_TASK_ID"] = {
"task-reference": "<parent>",
}
if "index" in task:
taskdesc["index"] = task["index"]
if task.get("cache", True) and not gecko_taskgraph.fast:
taskdesc["cache"] = {
"type": "docker-images.v2",
"name": image_name,
"digest-data": digest_data,
}
yield taskdesc