def build_context(name, outputFile, args=None): """Build a context.tar for image with specified name.""" if not name: raise ValueError("must provide a Docker image name") if not outputFile: raise ValueError("must provide a outputFile") image_dir = docker.image_path(name) if not os.path.isdir(image_dir): raise Exception("image directory does not exist: %s" % image_dir) docker.create_context_tar(".", image_dir, outputFile, args)
def test_create_context_missing_extra(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include does/not/exist\n') with self.assertRaisesRegexp(Exception, 'path does not exist'): docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') finally: shutil.rmtree(tmp)
def build_context(name, outputFile): """Build a context.tar for image with specified name. """ if not name: raise ValueError('must provide a Docker image name') if not outputFile: raise ValueError('must provide a outputFile') image_dir = os.path.join(docker.IMAGE_DIR, name) if not os.path.isdir(image_dir): raise Exception('image directory does not exist: %s' % image_dir) docker.create_context_tar(GECKO, image_dir, outputFile, "")
def test_create_context_outside_topsrcdir(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include foo/../../../etc/shadow\n") with self.assertRaisesRegexp(Exception, "path outside topsrcdir"): docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) finally: shutil.rmtree(tmp)
def test_create_context_missing_extra(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include does/not/exist\n") with self.assertRaisesRegexp(Exception, "path does not exist"): docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) finally: shutil.rmtree(tmp)
def build_context(name, outputFile): """Build a context.tar for image with specified name. """ if not name: raise ValueError('must provide a Docker image name') if not outputFile: raise ValueError('must provide a outputFile') image_dir = os.path.join(IMAGE_DIR, name) if not os.path.isdir(image_dir): raise Exception('image directory does not exist: %s' % image_dir) docker.create_context_tar(GECKO, image_dir, outputFile, "")
def test_create_context_outside_topsrcdir(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include foo/../../../etc/shadow\n') with self.assertRaisesRegexp(Exception, 'path outside topsrcdir'): docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') finally: shutil.rmtree(tmp)
def test_create_context_absolute_path(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) # Absolute paths in %include syntax are not allowed. with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include /etc/shadow\n") with self.assertRaisesRegexp(Exception, "cannot be absolute"): docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) finally: shutil.rmtree(tmp)
def test_create_context_absolute_path(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) # Absolute paths in %include syntax are not allowed. with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include /etc/shadow\n') with self.assertRaisesRegexp(Exception, 'cannot be absolute'): docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') finally: shutil.rmtree(tmp)
def test_create_context_topsrcdir_files(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra/file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) with open(os.path.join(extra, 'file0'), 'a'): pass os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'test_image', {}) self.assertEqual( h, '49dc3827530cd344d7bcc52e1fdd4aefc632568cf442cffd3dd9633a58f271bf' ) with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'Dockerfile', 'topsrcdir/extra/file0', ]) finally: shutil.rmtree(tmp)
def test_create_context_tar_basic(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test_image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'a'): pass os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) with open(os.path.join(d, 'extra'), 'a'): pass os.chmod(os.path.join(d, 'extra'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image', {}) self.assertEqual( h, 'eae3ad00936085eb3e5958912f79fb06ee8e14a91f7157c5f38625f7ddacb9c7' ) # File prefix should be "my_image" with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'Dockerfile', 'extra', ]) finally: shutil.rmtree(tmp)
def test_create_context_topsrcdir_files(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra/file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) with open(os.path.join(extra, 'file0'), 'a'): pass os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'test_image') self.assertEqual(h, 'e7f14044b8ec1ba42e251d4b293af212ad08b30ec8ab6613abbdbe73c3c2b61f') with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'Dockerfile', 'topsrcdir/extra/file0', ]) finally: shutil.rmtree(tmp)
def test_create_context_tar_basic(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test_image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'a'): pass os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) with open(os.path.join(d, 'extra'), 'a'): pass os.chmod(os.path.join(d, 'extra'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image') self.assertEqual( h, '2a6d7f1627eba60daf85402418e041d728827d309143c6bc1c6bb3035bde6717' ) # File prefix should be "my_image" with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'my_image/Dockerfile', 'my_image/extra', ]) finally: shutil.rmtree(tmp)
def test_create_context_topsrcdir_files(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra/file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) with open(os.path.join(extra, 'file0'), 'a'): pass os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'test_image') self.assertEqual( h, '20faeb7c134f21187b142b5fadba94ae58865dc929c6c293d8cbc0a087269338' ) with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'test_image/Dockerfile', 'test_image/topsrcdir/extra/file0', ]) finally: shutil.rmtree(tmp)
def test_create_context_tar_basic(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test_image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'a'): pass os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) with open(os.path.join(d, 'extra'), 'a'): pass os.chmod(os.path.join(d, 'extra'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image') self.assertEqual(h, '2a6d7f1627eba60daf85402418e041d728827d309143c6bc1c6bb3035bde6717') # File prefix should be "my_image" with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'my_image/Dockerfile', 'my_image/extra', ]) finally: shutil.rmtree(tmp)
def test_create_context_topsrcdir_files(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include extra/file0\n") os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) extra = os.path.join(tmp, "extra") os.mkdir(extra) with open(os.path.join(extra, "file0"), "a"): pass os.chmod(os.path.join(extra, "file0"), MODE_STANDARD) tp = os.path.join(tmp, "tar") h = docker.create_context_tar(tmp, d, tp, "test_image", {}) self.assertEqual( h, "49dc3827530cd344d7bcc52e1fdd4aefc632568cf442cffd3dd9633a58f271bf" ) with tarfile.open(tp, "r:gz") as tf: self.assertEqual( tf.getnames(), [ "Dockerfile", "topsrcdir/extra/file0", ], ) finally: shutil.rmtree(tmp)
def test_create_context_tar_basic(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test_image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "a"): pass os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) with open(os.path.join(d, "extra"), "a"): pass os.chmod(os.path.join(d, "extra"), MODE_STANDARD) tp = os.path.join(tmp, "tar") h = docker.create_context_tar(tmp, d, tp, "my_image", {}) self.assertEqual( h, "eae3ad00936085eb3e5958912f79fb06ee8e14a91f7157c5f38625f7ddacb9c7" ) # File prefix should be "my_image" with tarfile.open(tp, "r:gz") as tf: self.assertEqual( tf.getnames(), [ "Dockerfile", "extra", ], ) finally: shutil.rmtree(tmp)
def test_create_context_topsrcdir_files(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra/file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) with open(os.path.join(extra, 'file0'), 'a'): pass os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'test_image') self.assertEqual(h, '20faeb7c134f21187b142b5fadba94ae58865dc929c6c293d8cbc0a087269338') with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'test_image/Dockerfile', 'test_image/topsrcdir/extra/file0', ]) finally: shutil.rmtree(tmp)
def test_create_context_tar_basic(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test_image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'a'): pass os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) with open(os.path.join(d, 'extra'), 'a'): pass os.chmod(os.path.join(d, 'extra'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image') self.assertEqual(h, '6c1cc23357625f64f775a08eace7bbc3877dd08d2f3546e0f2e308bac8491865') # File prefix should be "my_image" with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'Dockerfile', 'extra', ]) finally: shutil.rmtree(tmp)
def build_image(name): """Build a Docker image of specified name. Output from image building process will be printed to stdout. """ if not name: raise ValueError('must provide a Docker image name') image_dir = os.path.join(IMAGE_DIR, name) if not os.path.isdir(image_dir): raise Exception('image directory does not exist: %s' % image_dir) tag = docker.docker_image(name, default_version='latest') docker_bin = which.which('docker') # Verify that Docker is working. try: subprocess.check_output([docker_bin, '--version']) except subprocess.CalledProcessError: raise Exception('Docker server is unresponsive. Run `docker ps` and ' 'check that Docker is running') # We obtain a context archive and build from that. Going through the # archive creation is important: it normalizes things like file owners # and mtimes to increase the chances that image generation is # deterministic. fd, context_path = tempfile.mkstemp() os.close(fd) try: docker.create_context_tar(GECKO, image_dir, context_path, name) docker.build_from_context(docker_bin, context_path, name, tag) finally: os.unlink(context_path) print('Successfully built %s and tagged with %s' % (name, tag)) if tag.endswith(':latest'): print('*' * 50) print('WARNING: no VERSION file found in image directory.') print('Image is not suitable for deploying/pushing.') print('Create an image suitable for deploying/pushing by creating') print('a VERSION file in the image directory.') print('*' * 50)
def build_image(name): """Build a Docker image of specified name. Output from image building process will be printed to stdout. """ if not name: raise ValueError('must provide a Docker image name') image_dir = os.path.join(docker.IMAGE_DIR, name) if not os.path.isdir(image_dir): raise Exception('image directory does not exist: %s' % image_dir) tag = docker.docker_image(name, by_tag=True) docker_bin = which.which('docker') # Verify that Docker is working. try: subprocess.check_output([docker_bin, '--version']) except subprocess.CalledProcessError: raise Exception('Docker server is unresponsive. Run `docker ps` and ' 'check that Docker is running') # We obtain a context archive and build from that. Going through the # archive creation is important: it normalizes things like file owners # and mtimes to increase the chances that image generation is # deterministic. fd, context_path = tempfile.mkstemp() os.close(fd) try: docker.create_context_tar(GECKO, image_dir, context_path, name) docker.build_from_context(docker_bin, context_path, name, tag) finally: os.unlink(context_path) print('Successfully built %s and tagged with %s' % (name, tag)) if tag.endswith(':latest'): print('*' * 50) print('WARNING: no VERSION file found in image directory.') print('Image is not suitable for deploying/pushing.') print('Create an image suitable for deploying/pushing by creating') print('a VERSION file in the image directory.') print('*' * 50)
def test_create_context_extra_directory(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, "test-image") os.mkdir(d) with open(os.path.join(d, "Dockerfile"), "wb") as fh: fh.write(b"# %include extra\n") fh.write(b"# %include file0\n") os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) extra = os.path.join(tmp, "extra") os.mkdir(extra) for i in range(3): p = os.path.join(extra, "file%d" % i) with open(p, "wb") as fh: fh.write(b"file%d" % i) os.chmod(p, MODE_STANDARD) with open(os.path.join(tmp, "file0"), "a"): pass os.chmod(os.path.join(tmp, "file0"), MODE_STANDARD) tp = os.path.join(tmp, "tar") h = docker.create_context_tar(tmp, d, tp, "my_image", {}) self.assertEqual( h, "a392f23cd6606ae43116390a4d0113354cff1e688a41d46f48b0fb25e90baa13" ) with tarfile.open(tp, "r:gz") as tf: self.assertEqual( tf.getnames(), [ "Dockerfile", "topsrcdir/extra/file0", "topsrcdir/extra/file1", "topsrcdir/extra/file2", "topsrcdir/file0", ], ) finally: shutil.rmtree(tmp)
def test_create_context_extra_directory(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra\n') fh.write(b'# %include file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) for i in range(3): p = os.path.join(extra, 'file%d' % i) with open(p, 'wb') as fh: fh.write(b'file%d' % i) os.chmod(p, MODE_STANDARD) with open(os.path.join(tmp, 'file0'), 'a'): pass os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image') self.assertEqual( h, 'e5440513ab46ae4c1d056269e1c6715d5da7d4bd673719d360411e35e5b87205' ) with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'my_image/Dockerfile', 'my_image/topsrcdir/extra/file0', 'my_image/topsrcdir/extra/file1', 'my_image/topsrcdir/extra/file2', 'my_image/topsrcdir/file0', ]) finally: shutil.rmtree(tmp)
def test_create_context_extra_directory(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra\n') fh.write(b'# %include file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) for i in range(3): p = os.path.join(extra, 'file%d' % i) with open(p, 'wb') as fh: fh.write(b'file%d' % i) os.chmod(p, MODE_STANDARD) with open(os.path.join(tmp, 'file0'), 'a'): pass os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image', {}) self.assertEqual( h, 'a392f23cd6606ae43116390a4d0113354cff1e688a41d46f48b0fb25e90baa13' ) with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'Dockerfile', 'topsrcdir/extra/file0', 'topsrcdir/extra/file1', 'topsrcdir/extra/file2', 'topsrcdir/file0', ]) finally: shutil.rmtree(tmp)
def test_create_context_extra_directory(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra\n') fh.write(b'# %include file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) for i in range(3): p = os.path.join(extra, 'file%d' % i) with open(p, 'wb') as fh: fh.write(b'file%d' % i) os.chmod(p, MODE_STANDARD) with open(os.path.join(tmp, 'file0'), 'a'): pass os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image') self.assertEqual(h, 'e5440513ab46ae4c1d056269e1c6715d5da7d4bd673719d360411e35e5b87205') with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'my_image/Dockerfile', 'my_image/topsrcdir/extra/file0', 'my_image/topsrcdir/extra/file1', 'my_image/topsrcdir/extra/file2', 'my_image/topsrcdir/file0', ]) finally: shutil.rmtree(tmp)
def test_create_context_extra_directory(self): tmp = tempfile.mkdtemp() try: d = os.path.join(tmp, 'test-image') os.mkdir(d) with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: fh.write(b'# %include extra\n') fh.write(b'# %include file0\n') os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) extra = os.path.join(tmp, 'extra') os.mkdir(extra) for i in range(3): p = os.path.join(extra, 'file%d' % i) with open(p, 'wb') as fh: fh.write(b'file%d' % i) os.chmod(p, MODE_STANDARD) with open(os.path.join(tmp, 'file0'), 'a'): pass os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD) tp = os.path.join(tmp, 'tar') h = docker.create_context_tar(tmp, d, tp, 'my_image') self.assertEqual(h, 'd2a3363b15d0eb547a6c81a72ddf3980e2f6e6360c29b4fb6818102896f43180') with tarfile.open(tp, 'r:gz') as tf: self.assertEqual(tf.getnames(), [ 'Dockerfile', 'topsrcdir/extra/file0', 'topsrcdir/extra/file1', 'topsrcdir/extra/file2', 'topsrcdir/file0', ]) finally: shutil.rmtree(tmp)
def fill_template(config, tasks): available_packages = set() for task in config.kind_dependencies_tasks: if task.kind != "packages": continue name = task.label.replace("packages-", "") available_packages.add(name) context_hashes = {} tasks = list(tasks) if not taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop("name") job_symbol = task.pop("symbol", None) args = task.pop("args", {}) definition = task.pop("definition", image_name) packages = task.pop("packages", []) parent = task.pop("parent", None) for p in packages: if p not in available_packages: raise Exception( "Missing package job for {}-{}: {}".format( config.kind, image_name, p ) ) if not taskgraph.fast: context_path = os.path.join("taskcluster", "docker", definition) topsrcdir = os.path.dirname(config.graph_config.taskcluster_yml) if config.write_artifacts: context_file = os.path.join(CONTEXTS_DIR, f"{image_name}.tar.gz") logger.info(f"Writing {context_file} for docker image {image_name}") context_hash = create_context_tar( topsrcdir, context_path, context_file, args, ) else: context_hash = generate_context_hash(topsrcdir, context_path, args) else: if config.write_artifacts: raise Exception("Can't write artifacts if `taskgraph.fast` is set.") context_hash = "0" * 40 digest_data = [context_hash] digest_data += [json.dumps(args, sort_keys=True)] context_hashes[image_name] = context_hash description = "Build the docker image {} for use by dependent tasks".format( image_name ) args["DOCKER_IMAGE_PACKAGES"] = " ".join(f"<{p}>" for p in packages) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = "3" if int(config.params["level"]) == 1 else "10" # include some information that is useful in reconstructing this task # from JSON taskdesc = { "label": "build-docker-image-" + image_name, "description": description, "attributes": { "image_name": image_name, "artifact_prefix": "public", }, "expires-after": "28 days" if config.params.is_try() else "1 year", "scopes": [], "run-on-projects": [], "worker-type": "images", "worker": { "implementation": "docker-worker", "os": "linux", "artifacts": [ { "type": "file", "path": "/workspace/image.tar.zst", "name": "public/image.tar.zst", } ], "env": { "CONTEXT_TASK_ID": {"task-reference": "<decision>"}, "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format( image_name ), "HASH": context_hash, "PROJECT": config.params["project"], "IMAGE_NAME": image_name, "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, "DOCKER_BUILD_ARGS": { "task-reference": json.dumps(args), }, "VCS_BASE_REPOSITORY": config.params["base_repository"], "VCS_HEAD_REPOSITORY": config.params["head_repository"], "VCS_HEAD_REV": config.params["head_rev"], "VCS_REPOSITORY_TYPE": config.params["repository_type"], }, "chain-of-trust": True, "max-run-time": 7200, }, } if "index" in task: taskdesc["index"] = task["index"] if job_symbol: taskdesc["treeherder"] = { "symbol": job_symbol, "platform": "taskcluster-images/opt", "kind": "other", "tier": 1, } worker = taskdesc["worker"] worker["docker-image"] = IMAGE_BUILDER_IMAGE digest_data.append(f"image-builder-image:{IMAGE_BUILDER_IMAGE}") if packages: deps = taskdesc.setdefault("dependencies", {}) for p in sorted(packages): deps[p] = f"packages-{p}" if parent: deps = taskdesc.setdefault("dependencies", {}) deps["parent"] = f"build-docker-image-{parent}" worker["env"]["PARENT_TASK_ID"] = { "task-reference": "<parent>", } if task.get("cache", True) and not taskgraph.fast: taskdesc["cache"] = { "type": "docker-images.v2", "name": image_name, "digest-data": digest_data, } yield taskdesc
def load_tasks(cls, kind, path, config, params, loaded_tasks): pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(params['pushdate'])) parameters = { 'pushlog_id': params.get('pushlog_id', 0), 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'project': params['project'], 'docker_image': docker_image, 'base_repository': params['base_repository'] or params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'owner': params['owner'], 'level': params['level'], 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'.format( repo=params['head_repository'], rev=params['head_rev']), } tasks = [] templates = Templates(path) for image_name in config['images']: context_path = os.path.join('testing', 'docker', image_name) image_parameters = dict(parameters) image_parameters['context_path'] = context_path image_parameters['artifact_path'] = 'public/image.tar' image_parameters['image_name'] = image_name image_artifact_path = \ "public/docker_image_contexts/{}/context.tar.gz".format(image_name) if os.environ.get('TASK_ID'): # We put image context tar balls in a different artifacts folder # on the Gecko decision task in order to have longer expiration # dates for smaller artifacts. destination = os.path.join( os.environ['HOME'], "docker_image_contexts/{}/context.tar.gz".format( image_name)) image_parameters['context_url'] = ARTIFACT_URL.format( os.environ['TASK_ID'], image_artifact_path) destination = os.path.abspath(destination) if not os.path.exists(os.path.dirname(destination)): os.makedirs(os.path.dirname(destination)) context_hash = create_context_tar(GECKO, context_path, destination, image_name) else: # skip context generation since this isn't a decision task # TODO: generate context tarballs using subdirectory clones in # the image-building task so we don't have to worry about this. image_parameters[ 'context_url'] = 'file:///tmp/' + image_artifact_path context_hash = generate_context_hash(GECKO, context_path, image_name) image_parameters['context_hash'] = context_hash image_task = templates.load('image.yml', image_parameters) attributes = {'image_name': image_name} # As an optimization, if the context hash exists for mozilla-central, that image # task ID will be used. The reasoning behind this is that eventually everything ends # up on mozilla-central at some point if most tasks use this as a common image # for a given context hash, a worker within Taskcluster does not need to contain # the same image per branch. index_paths = [ 'docker.images.v1.{}.{}.hash.{}'.format( project, image_name, context_hash) for project in ['mozilla-central', params['project']] ] tasks.append( cls(kind, 'build-docker-image-' + image_name, task=image_task['task'], attributes=attributes, index_paths=index_paths)) return tasks
def fill_template(config, tasks): if not taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop('name') job_symbol = task.pop('symbol') args = task.pop('args', {}) packages = task.pop('packages', []) parent = task.pop('parent', None) for p in packages: if "packages-{}".format(p) not in config.kind_dependencies_tasks: raise Exception('Missing package job for {}-{}: {}'.format( config.kind, image_name, p)) if not taskgraph.fast: context_path = mozpath.relpath(image_path(image_name), GECKO) if config.write_artifacts: context_file = os.path.join(CONTEXTS_DIR, '{}.tar.gz'.format(image_name)) logger.info("Writing {} for docker image {}".format( context_file, image_name)) context_hash = create_context_tar(GECKO, context_path, context_file, image_name, args) else: context_hash = generate_context_hash(GECKO, context_path, image_name, args) else: if config.write_artifacts: raise Exception( "Can't write artifacts if `taskgraph.fast` is set.") context_hash = '0' * 40 digest_data = [context_hash] digest_data += [json.dumps(args, sort_keys=True)] description = 'Build the docker image {} for use by dependent tasks'.format( image_name) args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p) for p in packages) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = '3' if int(config.params['level']) == 1 else '10' # include some information that is useful in reconstructing this task # from JSON taskdesc = { 'label': '{}-{}'.format(config.kind, image_name), 'description': description, 'attributes': { 'image_name': image_name, 'artifact_prefix': 'public', }, 'expires-after': '28 days' if config.params.is_try() else '1 year', 'scopes': [], 'treeherder': { 'symbol': job_symbol, 'platform': 'taskcluster-images/opt', 'kind': 'other', 'tier': 1, }, 'run-on-projects': [], 'worker-type': 'images', 'worker': { 'implementation': 'docker-worker', 'os': 'linux', 'artifacts': [{ 'type': 'file', 'path': '/workspace/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { 'CONTEXT_TASK_ID': { 'task-reference': "<decision>" }, 'CONTEXT_PATH': "public/docker-contexts/{}.tar.gz".format(image_name), 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'DOCKER_BUILD_ARGS': { 'task-reference': six.ensure_text(json.dumps(args)) }, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], }, 'chain-of-trust': True, 'max-run-time': 7200, # FIXME: We aren't currently propagating the exit code }, } # Retry for 'funsize-update-generator' if exit status code is -1 if image_name in ['funsize-update-generator']: taskdesc['worker']['retry-exit-status'] = [-1] worker = taskdesc['worker'] if image_name == 'image_builder': worker['docker-image'] = IMAGE_BUILDER_IMAGE digest_data.append( "image-builder-image:{}".format(IMAGE_BUILDER_IMAGE)) else: worker['docker-image'] = {'in-tree': 'image_builder'} deps = taskdesc.setdefault('dependencies', {}) deps['docker-image'] = '{}-image_builder'.format(config.kind) if packages: deps = taskdesc.setdefault('dependencies', {}) for p in sorted(packages): deps[p] = 'packages-{}'.format(p) if parent: deps = taskdesc.setdefault('dependencies', {}) deps['parent'] = '{}-{}'.format(config.kind, parent) worker['env']['PARENT_TASK_ID'] = { 'task-reference': '<parent>', } if 'index' in task: taskdesc['index'] = task['index'] if task.get('cache', True) and not taskgraph.fast: taskdesc['cache'] = { 'type': 'docker-images.v2', 'name': image_name, 'digest-data': digest_data, } yield taskdesc
def load_tasks(cls, kind, path, config, params, loaded_tasks): # TODO: make this match the pushdate (get it from a parameter rather than vcs) pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) parameters = { 'pushlog_id': params.get('pushlog_id', 0), 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'project': params['project'], 'docker_image': docker_image, 'base_repository': params['base_repository'] or params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'owner': params['owner'], 'level': params['level'], 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml' .format(repo=params['head_repository'], rev=params['head_rev']), } tasks = [] templates = Templates(path) for image_name in config['images']: context_path = os.path.join('testing', 'docker', image_name) image_parameters = dict(parameters) image_parameters['context_path'] = context_path image_parameters['artifact_path'] = 'public/image.tar' image_parameters['image_name'] = image_name image_artifact_path = \ "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name) if os.environ.get('TASK_ID'): destination = os.path.join( os.environ['HOME'], "artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name)) image_parameters['context_url'] = ARTIFACT_URL.format( os.environ['TASK_ID'], image_artifact_path) destination = os.path.abspath(destination) if not os.path.exists(os.path.dirname(destination)): os.makedirs(os.path.dirname(destination)) context_hash = create_context_tar(GECKO, context_path, destination, image_name) else: # skip context generation since this isn't a decision task # TODO: generate context tarballs using subdirectory clones in # the image-building task so we don't have to worry about this. image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path context_hash = generate_context_hash(GECKO, context_path, image_name) image_parameters['context_hash'] = context_hash image_task = templates.load('image.yml', image_parameters) attributes = {'image_name': image_name} # As an optimization, if the context hash exists for mozilla-central, that image # task ID will be used. The reasoning behind this is that eventually everything ends # up on mozilla-central at some point if most tasks use this as a common image # for a given context hash, a worker within Taskcluster does not need to contain # the same image per branch. index_paths = ['docker.images.v1.{}.{}.hash.{}'.format( project, image_name, context_hash) for project in ['mozilla-central', params['project']]] tasks.append(cls(kind, 'build-docker-image-' + image_name, task=image_task['task'], attributes=attributes, index_paths=index_paths)) return tasks
def fill_template(config, tasks): if not taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop("name") job_symbol = task.pop("symbol") args = task.pop("args", {}) packages = task.pop("packages", []) parent = task.pop("parent", None) for p in packages: if "packages-{}".format(p) not in config.kind_dependencies_tasks: raise Exception("Missing package job for {}-{}: {}".format( config.kind, image_name, p)) if not taskgraph.fast: context_path = mozpath.relpath(image_path(image_name), GECKO) if config.write_artifacts: context_file = os.path.join(CONTEXTS_DIR, "{}.tar.gz".format(image_name)) logger.info("Writing {} for docker image {}".format( context_file, image_name)) context_hash = create_context_tar(GECKO, context_path, context_file, image_name, args) else: context_hash = generate_context_hash(GECKO, context_path, image_name, args) else: if config.write_artifacts: raise Exception( "Can't write artifacts if `taskgraph.fast` is set.") context_hash = "0" * 40 digest_data = [context_hash] digest_data += [json.dumps(args, sort_keys=True)] description = "Build the docker image {} for use by dependent tasks".format( image_name) args["DOCKER_IMAGE_PACKAGES"] = " ".join("<{}>".format(p) for p in packages) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = "3" if int(config.params["level"]) == 1 else "10" # include some information that is useful in reconstructing this task # from JSON taskdesc = { "label": "{}-{}".format(config.kind, image_name), "description": description, "attributes": { "image_name": image_name, "artifact_prefix": "public", }, "expires-after": "1 year", "scopes": [], "treeherder": { "symbol": job_symbol, "platform": "taskcluster-images/opt", "kind": "other", "tier": 1, }, "run-on-projects": [], "worker-type": "images", "worker": { "implementation": "docker-worker", "os": "linux", "artifacts": [{ "type": "file", "path": "/workspace/image.tar.zst", "name": "public/image.tar.zst", }], "env": { "CONTEXT_TASK_ID": { "task-reference": "<decision>" }, "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format(image_name), "HASH": context_hash, "PROJECT": config.params["project"], "IMAGE_NAME": image_name, "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, "DOCKER_BUILD_ARGS": { "task-reference": six.ensure_text(json.dumps(args)) }, "GECKO_BASE_REPOSITORY": config.params["base_repository"], "GECKO_HEAD_REPOSITORY": config.params["head_repository"], "GECKO_HEAD_REV": config.params["head_rev"], }, "chain-of-trust": True, "max-run-time": 7200, # FIXME: We aren't currently propagating the exit code }, } # Retry for 'funsize-update-generator' if exit status code is -1 if image_name in ["funsize-update-generator"]: taskdesc["worker"]["retry-exit-status"] = [-1] worker = taskdesc["worker"] if image_name == "image_builder": worker["docker-image"] = IMAGE_BUILDER_IMAGE digest_data.append( "image-builder-image:{}".format(IMAGE_BUILDER_IMAGE)) else: worker["docker-image"] = {"in-tree": "image_builder"} deps = taskdesc.setdefault("dependencies", {}) deps["docker-image"] = "{}-image_builder".format(config.kind) if packages: deps = taskdesc.setdefault("dependencies", {}) for p in sorted(packages): deps[p] = "packages-{}".format(p) if parent: deps = taskdesc.setdefault("dependencies", {}) deps["parent"] = "{}-{}".format(config.kind, parent) worker["env"]["PARENT_TASK_ID"] = { "task-reference": "<parent>", } if "index" in task: taskdesc["index"] = task["index"] if task.get("cache", True) and not taskgraph.fast: taskdesc["cache"] = { "type": "docker-images.v2", "name": image_name, "digest-data": digest_data, } yield taskdesc