def test_create_context_missing_extra(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include does/not/exist\n") with self.assertRaisesRegexp(Exception, "path does not exist"): docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) finally: shutil.rmtree(tmp)
def test_create_context_outside_topsrcdir(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include foo/../../../etc/shadow\n") with self.assertRaisesRegexp(Exception, "path outside topsrcdir"): docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) finally: shutil.rmtree(tmp)
def test_create_context_absolute_path(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) # Absolute paths in %include syntax are not allowed. with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include /etc/shadow\n") with self.assertRaisesRegexp(Exception, "cannot be absolute"): docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) finally: shutil.rmtree(tmp)
def build_context(name, outputFile, args=None): """Build a context.tar for image with specified name.""" if not name: raise ValueError("must provide a Docker image name") if not outputFile: raise ValueError("must provide a outputFile") image_dir = docker.image_path(name) if not os.path.isdir(image_dir): raise Exception("image directory does not exist: %s" % image_dir) docker.create_context_tar(GECKO, image_dir, outputFile, image_name=name, args=args)
def test_create_context_tar_basic(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test_image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "a"): pass os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) with open(os.path.join(d, "extra"), "a"): pass os.chmod(os.path.join(d, "extra"), MODE_STANDARD) tp = os.path.join(tmp, "tar") h = docker.create_context_tar(tmp, d, tp, "my_image", {}) self.assertEqual( h, "eae3ad00936085eb3e5958912f79fb06ee8e14a91f7157c5f38625f7ddacb9c7" ) # File prefix should be "my_image" with tarfile.open(tp, "r:gz") as tf: self.assertEqual( tf.getnames(), [ "Dockerfile", "extra", ], ) finally: shutil.rmtree(tmp)
def test_create_context_topsrcdir_files(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include extra/file0\n") os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) extra = os.path.join(tmp, "extra") os.mkdir(extra) with open(os.path.join(extra, "file0"), "a"): pass os.chmod(os.path.join(extra, "file0"), MODE_STANDARD) tp = os.path.join(tmp, "tar") h = docker.create_context_tar(tmp, d, tp, "test_image", {}) self.assertEqual( h, "49dc3827530cd344d7bcc52e1fdd4aefc632568cf442cffd3dd9633a58f271bf" ) with tarfile.open(tp, "r:gz") as tf: self.assertEqual( tf.getnames(), [ "Dockerfile", "topsrcdir/extra/file0", ], ) finally: shutil.rmtree(tmp)
def test_create_context_extra_directory(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include extra\n") fh.write(b"# %include file0\n") os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) extra = os.path.join(tmp, "extra") os.mkdir(extra) for i in range(3): p = os.path.join(extra, "file%d" % i) with open(p, "wb") as fh: fh.write(b"file%d" % i) os.chmod(p, MODE_STANDARD) with open(os.path.join(tmp, "file0"), "a"): pass os.chmod(os.path.join(tmp, "file0"), MODE_STANDARD) tp = os.path.join(tmp, "tar") h = docker.create_context_tar(tmp, d, tp, "my_image", {}) self.assertEqual( h, "a392f23cd6606ae43116390a4d0113354cff1e688a41d46f48b0fb25e90baa13" ) with tarfile.open(tp, "r:gz") as tf: self.assertEqual( tf.getnames(), [ "Dockerfile", "topsrcdir/extra/file0", "topsrcdir/extra/file1", "topsrcdir/extra/file2", "topsrcdir/file0", ], ) finally: shutil.rmtree(tmp)
def fill_template(config, tasks): if not gecko_taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop("name") job_symbol = task.pop("symbol") args = task.pop("args", {}) packages = task.pop("packages", []) parent = task.pop("parent", None) for p in packages: if f"packages-{p}" not in config.kind_dependencies_tasks: raise Exception("Missing package job for {}-{}: {}".format( config.kind, image_name, p)) if not gecko_taskgraph.fast: context_path = mozpath.relpath(image_path(image_name), GECKO) if config.write_artifacts: context_file = os.path.join(CONTEXTS_DIR, f"{image_name}.tar.gz") logger.info( f"Writing {context_file} for docker image {image_name}") context_hash = create_context_tar(GECKO, context_path, context_file, image_name, args) else: context_hash = generate_context_hash(GECKO, context_path, image_name, args) else: if config.write_artifacts: raise Exception( "Can't write artifacts if `gecko_taskgraph.fast` is set.") context_hash = "0" * 40 digest_data = [context_hash] digest_data += [json.dumps(args, sort_keys=True)] description = "Build the docker image {} for use by dependent tasks".format( image_name) args["DOCKER_IMAGE_PACKAGES"] = " ".join(f"<{p}>" for p in packages) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = "3" if int(config.params["level"]) == 1 else "10" # include some information that is useful in reconstructing this task # from JSON taskdesc = { "label": f"{config.kind}-{image_name}", "description": description, "attributes": { "image_name": image_name, "artifact_prefix": "public", }, "expires-after": "1 year", "scopes": [], "treeherder": { "symbol": job_symbol, "platform": "taskcluster-images/opt", "kind": "other", "tier": 1, }, "run-on-projects": [], "worker-type": "images", "worker": { "implementation": "docker-worker", "os": "linux", "artifacts": [{ "type": "file", "path": "/workspace/image.tar.zst", "name": "public/image.tar.zst", }], "env": { "CONTEXT_TASK_ID": { "task-reference": "<decision>" }, "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format(image_name), "HASH": context_hash, "PROJECT": config.params["project"], "IMAGE_NAME": image_name, "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, "DOCKER_BUILD_ARGS": { "task-reference": json.dumps(args) }, "GECKO_BASE_REPOSITORY": config.params["base_repository"], "GECKO_HEAD_REPOSITORY": config.params["head_repository"], "GECKO_HEAD_REV": config.params["head_rev"], }, "chain-of-trust": True, "max-run-time": 7200, # FIXME: We aren't currently propagating the exit code }, } # Retry for 'funsize-update-generator' if exit status code is -1 if image_name in ["funsize-update-generator"]: taskdesc["worker"]["retry-exit-status"] = [-1] worker = taskdesc["worker"] if image_name == "image_builder": worker["docker-image"] = IMAGE_BUILDER_IMAGE digest_data.append(f"image-builder-image:{IMAGE_BUILDER_IMAGE}") else: worker["docker-image"] = {"in-tree": "image_builder"} deps = taskdesc.setdefault("dependencies", {}) deps["docker-image"] = f"{config.kind}-image_builder" if packages: deps = taskdesc.setdefault("dependencies", {}) for p in sorted(packages): deps[p] = f"packages-{p}" if parent: deps = taskdesc.setdefault("dependencies", {}) deps["parent"] = f"{config.kind}-{parent}" worker["env"]["PARENT_TASK_ID"] = { "task-reference": "<parent>", } if "index" in task: taskdesc["index"] = task["index"] if task.get("cache", True) and not gecko_taskgraph.fast: taskdesc["cache"] = { "type": "docker-images.v2", "name": image_name, "digest-data": digest_data, } yield taskdesc