def load_image_by_name(image_name, tag=None): context_path = os.path.join(GECKO, 'taskcluster', 'docker', image_name) context_hash = docker.generate_context_hash(GECKO, context_path, image_name) index_path = DOCKER_INDEX.format('level-3', image_name, context_hash) task_id = find_task_id(index_path) return load_image_by_task_id(task_id, tag)
def load_image_by_name(image_name, tag=None): context_path = os.path.join(GECKO, 'testing', 'docker', image_name) context_hash = docker.generate_context_hash(GECKO, context_path, image_name) image_index_url = INDEX_URL.format('level-3', image_name, context_hash) print("Fetching", image_index_url) task = json.load(urllib2.urlopen(image_index_url)) return load_image_by_task_id(task['taskId'], tag)
def load_image_by_name(image_name): context_path = os.path.join(GECKO, 'testing', 'docker', image_name) context_hash = docker.generate_context_hash(context_path) image_index_url = INDEX_URL.format('mozilla-central', image_name, context_hash) print("Fetching", image_index_url) task = json.load(urllib2.urlopen(image_index_url)) return load_image_by_task_id(task['taskId'])
def load_image_by_name(image_name, tag=None): context_path = os.path.join(GECKO, 'taskcluster', 'docker', image_name) context_hash = docker.generate_context_hash(GECKO, context_path, image_name) image_index_url = INDEX_URL.format('level-3', image_name, context_hash) print("Fetching", image_index_url) task = json.load(urllib2.urlopen(image_index_url)) return load_image_by_task_id(task['taskId'], tag)
def load_tasks(cls, kind, path, config, params, loaded_tasks): parameters = { 'pushlog_id': params.get('pushlog_id', 0), 'pushdate': params['moz_build_date'], 'pushtime': params['moz_build_date'][8:], 'year': params['moz_build_date'][0:4], 'month': params['moz_build_date'][4:6], 'day': params['moz_build_date'][6:8], 'project': params['project'], 'docker_image': docker_image, 'base_repository': params['base_repository'] or params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'owner': params['owner'], 'level': params['level'], 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml' .format(repo=params['head_repository'], rev=params['head_rev']), 'index_image_prefix': INDEX_PREFIX, 'artifact_path': 'public/image.tar.zst', } tasks = [] templates = Templates(path) for image_name, image_symbol in config['images'].iteritems(): context_path = os.path.join('taskcluster', 'docker', image_name) context_hash = generate_context_hash(GECKO, context_path, image_name) image_parameters = dict(parameters) image_parameters['image_name'] = image_name image_parameters['context_hash'] = context_hash image_task = templates.load('image.yml', image_parameters) attributes = {'image_name': image_name} # unique symbol for different docker image if 'extra' in image_task['task']: image_task['task']['extra']['treeherder']['symbol'] = image_symbol # As an optimization, if the context hash exists for a high level, that image # task ID will be used. The reasoning behind this is that eventually everything ends # up on level 3 at some point if most tasks use this as a common image # for a given context hash, a worker within Taskcluster does not need to contain # the same image per branch. index_paths = ['{}.level-{}.{}.hash.{}'.format( INDEX_PREFIX, level, image_name, context_hash) for level in range(int(params['level']), 4)] tasks.append(cls(kind, 'build-docker-image-' + image_name, task=image_task['task'], attributes=attributes, index_paths=index_paths)) return tasks
def load_image_by_name(image_name, tag=None): context_path = os.path.join(GECKO, 'taskcluster', 'docker', image_name) context_hash = docker.generate_context_hash(GECKO, context_path, image_name) index_path = cached_index_path( trust_domain='gecko', level=3, cache_type='docker-images.v1', cache_name=image_name, digest=context_hash, ) task_id = find_task_id(index_path) return load_image_by_task_id(task_id, tag)
def load_image_by_name(image_name, tag=None): context_path = os.path.join(GECKO, 'taskcluster', 'docker', image_name) context_hash = docker.generate_context_hash(GECKO, context_path, image_name) index_path = cached_index_path( trust_domain='gecko', level=3, cache_type='docker-images.v1', cache_name=image_name, digest=context_hash, ) task_id = find_task_id(index_path) return load_image_by_task_id(task_id, tag)
def test_generate_context_hash(self): tmpdir = tempfile.mkdtemp() try: os.makedirs(os.path.join(tmpdir, 'docker', 'my-image')) p = os.path.join(tmpdir, 'docker', 'my-image', 'Dockerfile') with open(p, 'w') as f: f.write("FROM node\nADD a-file\n") os.chmod(p, MODE_STANDARD) p = os.path.join(tmpdir, 'docker', 'my-image', 'a-file') with open(p, 'w') as f: f.write("data\n") os.chmod(p, MODE_STANDARD) self.assertEqual( docker.generate_context_hash( tmpdir, os.path.join(tmpdir, 'docker/my-image'), 'my-image'), 'e61e675ce05e8c11424437db3f1004079374c1a5fe6ad6800346cebe137b0797' ) finally: shutil.rmtree(tmpdir)
def test_generate_context_hash(self): tmpdir = tempfile.mkdtemp() try: os.makedirs(os.path.join(tmpdir, 'docker', 'my-image')) p = os.path.join(tmpdir, 'docker', 'my-image', 'Dockerfile') with open(p, 'w') as f: f.write("FROM node\nADD a-file\n") os.chmod(p, MODE_STANDARD) p = os.path.join(tmpdir, 'docker', 'my-image', 'a-file') with open(p, 'w') as f: f.write("data\n") os.chmod(p, MODE_STANDARD) self.assertEqual( docker.generate_context_hash(tmpdir, os.path.join(tmpdir, 'docker/my-image'), 'my-image'), 'e1649b3427bd7a0387f4508d25057c2e89228748517aad6c70e3df54f47bd13a' ) finally: shutil.rmtree(tmpdir)
def test_generate_context_hash(self): tmpdir = tempfile.mkdtemp() old_GECKO = docker.GECKO docker.GECKO = tmpdir try: os.makedirs(os.path.join(tmpdir, 'docker', 'my-image')) p = os.path.join(tmpdir, 'docker', 'my-image', 'Dockerfile') with open(p, 'w') as f: f.write("FROM node\nADD a-file\n") os.chmod(p, MODE_STANDARD) p = os.path.join(tmpdir, 'docker', 'my-image', 'a-file') with open(p, 'w') as f: f.write("data\n") os.chmod(p, MODE_STANDARD) self.assertEqual( docker.generate_context_hash(docker.GECKO, os.path.join(docker.GECKO, 'docker/my-image'), 'my-image'), 'e61e675ce05e8c11424437db3f1004079374c1a5fe6ad6800346cebe137b0797' ) finally: docker.GECKO = old_GECKO shutil.rmtree(tmpdir)
def test_generate_context_hash(self): tmpdir = tempfile.mkdtemp() try: os.makedirs(os.path.join(tmpdir, "docker", "my-image")) p = os.path.join(tmpdir, "docker", "my-image", "Dockerfile") with open(p, "w") as f: f.write("FROM node\nADD a-file\n") os.chmod(p, MODE_STANDARD) p = os.path.join(tmpdir, "docker", "my-image", "a-file") with open(p, "w") as f: f.write("data\n") os.chmod(p, MODE_STANDARD) self.assertEqual( docker.generate_context_hash( tmpdir, os.path.join(tmpdir, "docker/my-image"), "my-image", {}, ), "680532a33c845e3b4f8ea8a7bd697da579b647f28c29f7a0a71e51e6cca33983", ) finally: shutil.rmtree(tmpdir)
def fill_template(config, tasks): available_packages = set() for task in config.kind_dependencies_tasks: if task.kind != 'packages': continue name = task.label.replace('packages-', '') available_packages.add(name) context_hashes = {} for task in order_image_tasks(config, tasks): image_name = task.pop('name') job_symbol = task.pop('symbol') args = task.pop('args', {}) definition = task.pop('definition', image_name) packages = task.pop('packages', []) parent = task.pop('parent', None) for p in packages: if p not in available_packages: raise Exception('Missing package job for {}-{}: {}'.format( config.kind, image_name, p)) # Generating the context hash relies on arguments being set, so we # set this now, although it's not the final value (it's a # task-reference value, see further below). We add the package routes # containing a hash to get the overall docker image hash, so changes # to packages will be reflected in the docker image hash. args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p) for p in packages) if parent: args['DOCKER_IMAGE_PARENT'] = '{}:{}'.format( parent, context_hashes[parent]) args['TASKCLUSTER_ROOT_URL'] = get_root_url(False) if not taskgraph.fast: context_path = os.path.join('taskcluster', 'docker', definition) context_hash = generate_context_hash(GECKO, context_path, image_name, args) else: context_hash = '0' * 40 digest_data = [context_hash] context_hashes[image_name] = context_hash description = 'Build the docker image {} for use by dependent tasks'.format( image_name) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = '3' if int(config.params['level']) == 1 else '10' # include some information that is useful in reconstructing this task # from JSON taskdesc = { 'label': 'build-docker-image-' + image_name, 'description': description, 'attributes': { 'image_name': image_name }, 'expires-after': '28 days' if config.params.is_try() else '1 year', 'scopes': [ 'secrets:get:project/taskcluster/gecko/hgfingerprint', 'secrets:get:project/taskcluster/gecko/hgmointernal', ], 'treeherder': { 'symbol': job_symbol, 'platform': 'taskcluster-images/opt', 'kind': 'other', 'tier': 1, }, 'run-on-projects': [], 'worker-type': 'images', 'worker': { 'implementation': 'docker-worker', 'os': 'linux', 'artifacts': [{ 'type': 'file', 'path': '/builds/worker/workspace/artifacts/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { 'HG_STORE_PATH': '/builds/worker/checkouts/hg-store', 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], }, 'chain-of-trust': True, 'docker-in-docker': True, 'taskcluster-proxy': True, 'max-run-time': 7200, }, } # Retry for 'funsize-update-generator' if exit status code is -1 if image_name in ['funsize-update-generator']: taskdesc['worker']['retry-exit-status'] = [-1] worker = taskdesc['worker'] # We use the in-tree image_builder image to build docker images, but # that can't be used to build the image_builder image itself, # obviously. So we fall back to an image on docker hub, identified # by hash. After the image-builder image is updated, it's best to push # and update this hash as well, to keep image-builder builds up to date. if image_name == 'image_builder': hash = 'sha256:c6622fd3e5794842ad83d129850330b26e6ba671e39c58ee288a616a3a1c4c73' worker['docker-image'] = 'taskcluster/image_builder@' + hash # Keep in sync with the Dockerfile used to generate the # docker image whose digest is referenced above. worker['volumes'] = [ '/builds/worker/checkouts', '/builds/worker/workspace', ] cache_name = 'imagebuilder-v1' else: worker['docker-image'] = {'in-tree': 'image_builder'} cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix()) # Force images built against the in-tree image builder to # have a different digest by adding a fixed string to the # hashed data. # Append to this data whenever the image builder's output behavior # is changed, in order to force all downstream images to be rebuilt and # cached distinctly. digest_data.append('image_builder') # Updated for squashing images in Bug 1527394 digest_data.append('squashing layers') worker['caches'] = [{ 'type': 'persistent', 'name': cache_name, 'mount-point': '/builds/worker/checkouts', }] for k, v in args.items(): if k == 'DOCKER_IMAGE_PACKAGES': worker['env'][k] = {'task-reference': v} else: worker['env'][k] = v if packages: deps = taskdesc.setdefault('dependencies', {}) for p in sorted(packages): deps[p] = 'packages-{}'.format(p) if parent: deps = taskdesc.setdefault('dependencies', {}) deps['parent'] = 'build-docker-image-{}'.format(parent) worker['env']['DOCKER_IMAGE_PARENT_TASK'] = { 'task-reference': '<parent>', } if 'index' in task: taskdesc['index'] = task['index'] if task.get('cache', True) and not taskgraph.fast: taskdesc['cache'] = { 'type': 'docker-images.v2', 'name': image_name, 'digest-data': digest_data, } yield taskdesc
def load_tasks(cls, kind, path, config, params, loaded_tasks): # TODO: make this match the pushdate (get it from a parameter rather than vcs) pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) parameters = { 'pushlog_id': params.get('pushlog_id', 0), 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'project': params['project'], 'docker_image': docker_image, 'base_repository': params['base_repository'] or params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'owner': params['owner'], 'level': params['level'], 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml' .format(repo=params['head_repository'], rev=params['head_rev']), } tasks = [] templates = Templates(path) for image_name in config['images']: context_path = os.path.join('testing', 'docker', image_name) context_hash = generate_context_hash(context_path) image_parameters = dict(parameters) image_parameters['context_hash'] = context_hash image_parameters['context_path'] = context_path image_parameters['artifact_path'] = 'public/image.tar' image_parameters['image_name'] = image_name image_artifact_path = \ "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name) if os.environ.get('TASK_ID'): destination = os.path.join( os.environ['HOME'], "artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name)) image_parameters['context_url'] = ARTIFACT_URL.format( os.environ['TASK_ID'], image_artifact_path) cls.create_context_tar(context_path, destination, image_name) else: # skip context generation since this isn't a decision task # TODO: generate context tarballs using subdirectory clones in # the image-building task so we don't have to worry about this. image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path image_task = templates.load('image.yml', image_parameters) attributes = {'image_name': image_name} # As an optimization, if the context hash exists for mozilla-central, that image # task ID will be used. The reasoning behind this is that eventually everything ends # up on mozilla-central at some point if most tasks use this as a common image # for a given context hash, a worker within Taskcluster does not need to contain # the same image per branch. index_paths = ['docker.images.v1.{}.{}.hash.{}'.format( project, image_name, context_hash) for project in ['mozilla-central', params['project']]] tasks.append(cls(kind, 'build-docker-image-' + image_name, task=image_task['task'], attributes=attributes, index_paths=index_paths)) return tasks
def fill_template(config, tasks): if not taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop("name") job_symbol = task.pop("symbol") args = task.pop("args", {}) packages = task.pop("packages", []) parent = task.pop("parent", None) for p in packages: if "packages-{}".format(p) not in config.kind_dependencies_tasks: raise Exception("Missing package job for {}-{}: {}".format( config.kind, image_name, p)) if not taskgraph.fast: context_path = mozpath.relpath(image_path(image_name), GECKO) if config.write_artifacts: context_file = os.path.join(CONTEXTS_DIR, "{}.tar.gz".format(image_name)) logger.info("Writing {} for docker image {}".format( context_file, image_name)) context_hash = create_context_tar(GECKO, context_path, context_file, image_name, args) else: context_hash = generate_context_hash(GECKO, context_path, image_name, args) else: if config.write_artifacts: raise Exception( "Can't write artifacts if `taskgraph.fast` is set.") context_hash = "0" * 40 digest_data = [context_hash] digest_data += [json.dumps(args, sort_keys=True)] description = "Build the docker image {} for use by dependent tasks".format( image_name) args["DOCKER_IMAGE_PACKAGES"] = " ".join("<{}>".format(p) for p in packages) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = "3" if int(config.params["level"]) == 1 else "10" # include some information that is useful in reconstructing this task # from JSON taskdesc = { "label": "{}-{}".format(config.kind, image_name), "description": description, "attributes": { "image_name": image_name, "artifact_prefix": "public", }, "expires-after": "1 year", "scopes": [], "treeherder": { "symbol": job_symbol, "platform": "taskcluster-images/opt", "kind": "other", "tier": 1, }, "run-on-projects": [], "worker-type": "images", "worker": { "implementation": "docker-worker", "os": "linux", "artifacts": [{ "type": "file", "path": "/workspace/image.tar.zst", "name": "public/image.tar.zst", }], "env": { "CONTEXT_TASK_ID": { "task-reference": "<decision>" }, "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format(image_name), "HASH": context_hash, "PROJECT": config.params["project"], "IMAGE_NAME": image_name, "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, "DOCKER_BUILD_ARGS": { "task-reference": six.ensure_text(json.dumps(args)) }, "GECKO_BASE_REPOSITORY": config.params["base_repository"], "GECKO_HEAD_REPOSITORY": config.params["head_repository"], "GECKO_HEAD_REV": config.params["head_rev"], }, "chain-of-trust": True, "max-run-time": 7200, # FIXME: We aren't currently propagating the exit code }, } # Retry for 'funsize-update-generator' if exit status code is -1 if image_name in ["funsize-update-generator"]: taskdesc["worker"]["retry-exit-status"] = [-1] worker = taskdesc["worker"] if image_name == "image_builder": worker["docker-image"] = IMAGE_BUILDER_IMAGE digest_data.append( "image-builder-image:{}".format(IMAGE_BUILDER_IMAGE)) else: worker["docker-image"] = {"in-tree": "image_builder"} deps = taskdesc.setdefault("dependencies", {}) deps["docker-image"] = "{}-image_builder".format(config.kind) if packages: deps = taskdesc.setdefault("dependencies", {}) for p in sorted(packages): deps[p] = "packages-{}".format(p) if parent: deps = taskdesc.setdefault("dependencies", {}) deps["parent"] = "{}-{}".format(config.kind, parent) worker["env"]["PARENT_TASK_ID"] = { "task-reference": "<parent>", } if "index" in task: taskdesc["index"] = task["index"] if task.get("cache", True) and not taskgraph.fast: taskdesc["cache"] = { "type": "docker-images.v2", "name": image_name, "digest-data": digest_data, } yield taskdesc
def fill_template(config, tasks): available_packages = set() for task in config.kind_dependencies_tasks: if task.kind != "packages": continue name = task.label.replace("packages-", "") available_packages.add(name) context_hashes = {} tasks = list(tasks) if not taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop("name") job_symbol = task.pop("symbol", None) args = task.pop("args", {}) definition = task.pop("definition", image_name) packages = task.pop("packages", []) parent = task.pop("parent", None) for p in packages: if p not in available_packages: raise Exception( "Missing package job for {}-{}: {}".format( config.kind, image_name, p ) ) if not taskgraph.fast: context_path = os.path.join("taskcluster", "docker", definition) topsrcdir = os.path.dirname(config.graph_config.taskcluster_yml) if config.write_artifacts: context_file = os.path.join(CONTEXTS_DIR, f"{image_name}.tar.gz") logger.info(f"Writing {context_file} for docker image {image_name}") context_hash = create_context_tar( topsrcdir, context_path, context_file, args, ) else: context_hash = generate_context_hash(topsrcdir, context_path, args) else: if config.write_artifacts: raise Exception("Can't write artifacts if `taskgraph.fast` is set.") context_hash = "0" * 40 digest_data = [context_hash] digest_data += [json.dumps(args, sort_keys=True)] context_hashes[image_name] = context_hash description = "Build the docker image {} for use by dependent tasks".format( image_name ) args["DOCKER_IMAGE_PACKAGES"] = " ".join(f"<{p}>" for p in packages) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = "3" if int(config.params["level"]) == 1 else "10" # include some information that is useful in reconstructing this task # from JSON taskdesc = { "label": "build-docker-image-" + image_name, "description": description, "attributes": { "image_name": image_name, "artifact_prefix": "public", }, "expires-after": "28 days" if config.params.is_try() else "1 year", "scopes": [], "run-on-projects": [], "worker-type": "images", "worker": { "implementation": "docker-worker", "os": "linux", "artifacts": [ { "type": "file", "path": "/workspace/image.tar.zst", "name": "public/image.tar.zst", } ], "env": { "CONTEXT_TASK_ID": {"task-reference": "<decision>"}, "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format( image_name ), "HASH": context_hash, "PROJECT": config.params["project"], "IMAGE_NAME": image_name, "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, "DOCKER_BUILD_ARGS": { "task-reference": json.dumps(args), }, "VCS_BASE_REPOSITORY": config.params["base_repository"], "VCS_HEAD_REPOSITORY": config.params["head_repository"], "VCS_HEAD_REV": config.params["head_rev"], "VCS_REPOSITORY_TYPE": config.params["repository_type"], }, "chain-of-trust": True, "max-run-time": 7200, }, } if "index" in task: taskdesc["index"] = task["index"] if job_symbol: taskdesc["treeherder"] = { "symbol": job_symbol, "platform": "taskcluster-images/opt", "kind": "other", "tier": 1, } worker = taskdesc["worker"] worker["docker-image"] = IMAGE_BUILDER_IMAGE digest_data.append(f"image-builder-image:{IMAGE_BUILDER_IMAGE}") if packages: deps = taskdesc.setdefault("dependencies", {}) for p in sorted(packages): deps[p] = f"packages-{p}" if parent: deps = taskdesc.setdefault("dependencies", {}) deps["parent"] = f"build-docker-image-{parent}" worker["env"]["PARENT_TASK_ID"] = { "task-reference": "<parent>", } if task.get("cache", True) and not taskgraph.fast: taskdesc["cache"] = { "type": "docker-images.v2", "name": image_name, "digest-data": digest_data, } yield taskdesc
def fill_template(config, tasks): for task in tasks: image_name = task.pop('name') job_symbol = task.pop('symbol') context_path = os.path.join('taskcluster', 'docker', image_name) context_hash = generate_context_hash(GECKO, context_path, image_name) description = 'Build the docker image {} for use by dependent tasks'.format( image_name) routes = [] for tpl in ROUTE_TEMPLATES: routes.append(tpl.format( index_prefix=INDEX_PREFIX, level=config.params['level'], image_name=image_name, project=config.params['project'], head_rev=config.params['head_rev'], pushlog_id=config.params.get('pushlog_id', 0), pushtime=config.params['moz_build_date'][8:], year=config.params['moz_build_date'][0:4], month=config.params['moz_build_date'][4:6], day=config.params['moz_build_date'][6:8], context_hash=context_hash, )) # As an optimization, if the context hash exists for a high level, that image # task ID will be used. The reasoning behind this is that eventually everything ends # up on level 3 at some point if most tasks use this as a common image # for a given context hash, a worker within Taskcluster does not need to contain # the same image per branch. optimizations = [['index-search', '{}.level-{}.{}.hash.{}'.format( INDEX_PREFIX, level, image_name, context_hash)] for level in reversed(range(int(config.params['level']), 4))] # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = '3' if int(config.params['level']) == 1 else '10' # include some information that is useful in reconstructing this task # from JSON taskdesc = { 'label': 'build-docker-image-' + image_name, 'description': description, 'attributes': {'image_name': image_name}, 'expires-after': '1 year', 'routes': routes, 'optimizations': optimizations, 'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'], 'treeherder': { 'symbol': job_symbol, 'platform': 'taskcluster-images/opt', 'kind': 'other', 'tier': 1, }, 'run-on-projects': [], 'worker-type': 'aws-provisioner-v1/gecko-images', # can't use {in-tree: ..} here, otherwise we might try to build # this image.. 'worker': { 'implementation': 'docker-worker', 'docker-image': docker_image('image_builder'), 'caches': [{ 'type': 'persistent', 'name': 'level-{}-imagebuilder-v1'.format(config.params['level']), 'mount-point': '/home/worker/checkouts', }], 'artifacts': [{ 'type': 'file', 'path': '/home/worker/workspace/artifacts/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { 'HG_STORE_PATH': '/home/worker/checkouts/hg-store', 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], }, 'chain-of-trust': True, 'docker-in-docker': True, 'taskcluster-proxy': True, 'max-run-time': 3600, }, } yield taskdesc
def fill_template(config, tasks): for task in tasks: image_name = task.pop('name') job_symbol = task.pop('symbol') context_path = os.path.join('taskcluster', 'docker', image_name) context_hash = generate_context_hash(GECKO, context_path, image_name) description = 'Build the docker image {} for use by dependent tasks'.format( image_name) routes = [] for tpl in ROUTE_TEMPLATES: routes.append( tpl.format( index_prefix=INDEX_PREFIX, level=config.params['level'], image_name=image_name, project=config.params['project'], head_rev=config.params['head_rev'], pushlog_id=config.params.get('pushlog_id', 0), pushtime=config.params['moz_build_date'][8:], year=config.params['moz_build_date'][0:4], month=config.params['moz_build_date'][4:6], day=config.params['moz_build_date'][6:8], context_hash=context_hash, )) # As an optimization, if the context hash exists for a high level, that image # task ID will be used. The reasoning behind this is that eventually everything ends # up on level 3 at some point if most tasks use this as a common image # for a given context hash, a worker within Taskcluster does not need to contain # the same image per branch. optimizations = [[ 'index-search', '{}.level-{}.{}.hash.{}'.format(INDEX_PREFIX, level, image_name, context_hash) ] for level in reversed(range(int(config.params['level']), 4))] # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = '3' if int(config.params['level']) == 1 else '10' # include some information that is useful in reconstructing this task # from JSON taskdesc = { 'label': 'build-docker-image-' + image_name, 'description': description, 'attributes': { 'image_name': image_name }, 'expires-after': '1 year', 'routes': routes, 'optimizations': optimizations, 'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'], 'treeherder': { 'symbol': job_symbol, 'platform': 'taskcluster-images/opt', 'kind': 'other', 'tier': 1, }, 'run-on-projects': [], 'worker-type': 'aws-provisioner-v1/gecko-{}-images'.format( config.params['level']), # can't use {in-tree: ..} here, otherwise we might try to build # this image.. 'worker': { 'implementation': 'docker-worker', 'os': 'linux', 'docker-image': docker_image('image_builder'), 'caches': [{ 'type': 'persistent', 'name': 'level-{}-imagebuilder-v1'.format(config.params['level']), 'mount-point': '/builds/worker/checkouts', }], 'volumes': [ # Keep in sync with Dockerfile and TASKCLUSTER_VOLUMES '/builds/worker/checkouts', '/builds/worker/workspace', ], 'artifacts': [{ 'type': 'file', 'path': '/builds/worker/workspace/artifacts/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { 'HG_STORE_PATH': '/builds/worker/checkouts/hg-store', 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], 'TASKCLUSTER_VOLUMES': '/builds/worker/checkouts;/builds/worker/workspace', }, 'chain-of-trust': True, 'docker-in-docker': True, 'taskcluster-proxy': True, 'max-run-time': 7200, }, } yield taskdesc
def fill_template(config, tasks): if not taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop('name') job_symbol = task.pop('symbol') args = task.pop('args', {}) packages = task.pop('packages', []) parent = task.pop('parent', None) for p in packages: if "packages-{}".format(p) not in config.kind_dependencies_tasks: raise Exception('Missing package job for {}-{}: {}'.format( config.kind, image_name, p)) if not taskgraph.fast: context_path = mozpath.relpath(image_path(image_name), GECKO) if config.write_artifacts: context_file = os.path.join(CONTEXTS_DIR, '{}.tar.gz'.format(image_name)) logger.info("Writing {} for docker image {}".format( context_file, image_name)) context_hash = create_context_tar(GECKO, context_path, context_file, image_name, args) else: context_hash = generate_context_hash(GECKO, context_path, image_name, args) else: if config.write_artifacts: raise Exception( "Can't write artifacts if `taskgraph.fast` is set.") context_hash = '0' * 40 digest_data = [context_hash] digest_data += [json.dumps(args, sort_keys=True)] description = 'Build the docker image {} for use by dependent tasks'.format( image_name) args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p) for p in packages) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = '3' if int(config.params['level']) == 1 else '10' # include some information that is useful in reconstructing this task # from JSON taskdesc = { 'label': '{}-{}'.format(config.kind, image_name), 'description': description, 'attributes': { 'image_name': image_name, 'artifact_prefix': 'public', }, 'expires-after': '28 days' if config.params.is_try() else '1 year', 'scopes': [], 'treeherder': { 'symbol': job_symbol, 'platform': 'taskcluster-images/opt', 'kind': 'other', 'tier': 1, }, 'run-on-projects': [], 'worker-type': 'images', 'worker': { 'implementation': 'docker-worker', 'os': 'linux', 'artifacts': [{ 'type': 'file', 'path': '/workspace/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { 'CONTEXT_TASK_ID': { 'task-reference': "<decision>" }, 'CONTEXT_PATH': "public/docker-contexts/{}.tar.gz".format(image_name), 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'DOCKER_BUILD_ARGS': { 'task-reference': six.ensure_text(json.dumps(args)) }, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], }, 'chain-of-trust': True, 'max-run-time': 7200, # FIXME: We aren't currently propagating the exit code }, } # Retry for 'funsize-update-generator' if exit status code is -1 if image_name in ['funsize-update-generator']: taskdesc['worker']['retry-exit-status'] = [-1] worker = taskdesc['worker'] if image_name == 'image_builder': worker['docker-image'] = IMAGE_BUILDER_IMAGE digest_data.append( "image-builder-image:{}".format(IMAGE_BUILDER_IMAGE)) else: worker['docker-image'] = {'in-tree': 'image_builder'} deps = taskdesc.setdefault('dependencies', {}) deps['docker-image'] = '{}-image_builder'.format(config.kind) if packages: deps = taskdesc.setdefault('dependencies', {}) for p in sorted(packages): deps[p] = 'packages-{}'.format(p) if parent: deps = taskdesc.setdefault('dependencies', {}) deps['parent'] = '{}-{}'.format(config.kind, parent) worker['env']['PARENT_TASK_ID'] = { 'task-reference': '<parent>', } if 'index' in task: taskdesc['index'] = task['index'] if task.get('cache', True) and not taskgraph.fast: taskdesc['cache'] = { 'type': 'docker-images.v2', 'name': image_name, 'digest-data': digest_data, } yield taskdesc
def load_tasks(cls, kind, path, config, params, loaded_tasks): # TODO: make this match the pushdate (get it from a parameter rather than vcs) pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) parameters = { 'pushlog_id': params.get('pushlog_id', 0), 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'project': params['project'], 'docker_image': docker_image, 'base_repository': params['base_repository'] or params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'owner': params['owner'], 'level': params['level'], 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'.format( repo=params['head_repository'], rev=params['head_rev']), } tasks = [] templates = Templates(path) for image_name in config['images']: context_path = os.path.join('testing', 'docker', image_name) context_hash = generate_context_hash(context_path) image_parameters = dict(parameters) image_parameters['context_hash'] = context_hash image_parameters['context_path'] = context_path image_parameters['artifact_path'] = 'public/image.tar' image_parameters['image_name'] = image_name image_artifact_path = \ "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name) if os.environ.get('TASK_ID'): destination = os.path.join( os.environ['HOME'], "artifacts/decision_task/image_contexts/{}/context.tar.gz". format(image_name)) image_parameters['context_url'] = ARTIFACT_URL.format( os.environ['TASK_ID'], image_artifact_path) cls.create_context_tar(context_path, destination, image_name) else: # skip context generation since this isn't a decision task # TODO: generate context tarballs using subdirectory clones in # the image-building task so we don't have to worry about this. image_parameters[ 'context_url'] = 'file:///tmp/' + image_artifact_path image_task = templates.load('image.yml', image_parameters) attributes = {'image_name': image_name} # As an optimization, if the context hash exists for mozilla-central, that image # task ID will be used. The reasoning behind this is that eventually everything ends # up on mozilla-central at some point if most tasks use this as a common image # for a given context hash, a worker within Taskcluster does not need to contain # the same image per branch. index_paths = [ 'docker.images.v1.{}.{}.hash.{}'.format( project, image_name, context_hash) for project in ['mozilla-central', params['project']] ] tasks.append( cls(kind, 'build-docker-image-' + image_name, task=image_task['task'], attributes=attributes, index_paths=index_paths)) return tasks
def fill_template(config, tasks): available_packages = {} for task in config.kind_dependencies_tasks: if task.kind != 'packages': continue name = task.label.replace('packages-', '') for route in task.task.get('routes', []): if route.startswith('index.') and '.hash.' in route: # Only keep the hash part of the route. h = route.rsplit('.', 1)[1] assert DIGEST_RE.match(h) available_packages[name] = h break context_hashes = {} for task in order_image_tasks(config, tasks): image_name = task.pop('name') job_symbol = task.pop('symbol') args = task.pop('args', {}) definition = task.pop('definition', image_name) packages = task.pop('packages', []) parent = task.pop('parent', None) for p in packages: if p not in available_packages: raise Exception('Missing package job for {}-{}: {}'.format( config.kind, image_name, p)) # Generating the context hash relies on arguments being set, so we # set this now, although it's not the final value (it's a # task-reference value, see further below). We add the package routes # containing a hash to get the overall docker image hash, so changes # to packages will be reflected in the docker image hash. args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p) for p in packages) if parent: args['DOCKER_IMAGE_PARENT'] = '{}:{}'.format( parent, context_hashes[parent]) context_path = os.path.join('taskcluster', 'docker', definition) context_hash = generate_context_hash(GECKO, context_path, image_name, args) digest_data = [context_hash] context_hashes[image_name] = context_hash description = 'Build the docker image {} for use by dependent tasks'.format( image_name) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = '3' if int(config.params['level']) == 1 else '10' # include some information that is useful in reconstructing this task # from JSON taskdesc = { 'label': 'build-docker-image-' + image_name, 'description': description, 'attributes': { 'image_name': image_name }, 'expires-after': '28 days' if config.params.is_try() else '1 year', 'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'], 'treeherder': { 'symbol': job_symbol, 'platform': 'taskcluster-images/opt', 'kind': 'other', 'tier': 1, }, 'run-on-projects': [], 'worker-type': 'aws-provisioner-v1/gecko-{}-images'.format( config.params['level']), 'worker': { 'implementation': 'docker-worker', 'os': 'linux', 'artifacts': [{ 'type': 'file', 'path': '/builds/worker/workspace/artifacts/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { 'HG_STORE_PATH': '/builds/worker/checkouts/hg-store', 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], }, 'chain-of-trust': True, 'docker-in-docker': True, 'taskcluster-proxy': True, 'max-run-time': 7200, }, } # Retry for 'funsize-update-generator' if exit status code is -1 if image_name in ['funsize-update-generator']: taskdesc['worker']['retry-exit-status'] = [-1] worker = taskdesc['worker'] # We use the in-tree image_builder image to build docker images, but # that can't be used to build the image_builder image itself, # obviously. So we fall back to the last snapshot of the image that # was uploaded to docker hub. if image_name == 'image_builder': worker['docker-image'] = 'taskcluster/image_builder@sha256:' + \ '24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315' # Keep in sync with the Dockerfile used to generate the # docker image whose digest is referenced above. worker['volumes'] = [ '/builds/worker/checkouts', '/builds/worker/workspace', ] cache_name = 'imagebuilder-v1' else: worker['docker-image'] = {'in-tree': 'image_builder'} cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix()) # Force images built against the in-tree image builder to # have a different digest by adding a fixed string to the # hashed data. digest_data.append('image_builder') worker['caches'] = [{ 'type': 'persistent', 'name': 'level-{}-{}'.format(config.params['level'], cache_name), 'mount-point': '/builds/worker/checkouts', }] for k, v in args.items(): if k == 'DOCKER_IMAGE_PACKAGES': worker['env'][k] = {'task-reference': v} else: worker['env'][k] = v if packages: deps = taskdesc.setdefault('dependencies', {}) for p in sorted(packages): deps[p] = 'packages-{}'.format(p) digest_data.append(available_packages[p]) if parent: deps = taskdesc.setdefault('dependencies', {}) deps[parent] = 'build-docker-image-{}'.format(parent) worker['env']['DOCKER_IMAGE_PARENT_TASK'] = { 'task-reference': '<{}>'.format(parent) } if len(digest_data) > 1: kwargs = {'digest_data': digest_data} else: kwargs = {'digest': digest_data[0]} add_optimization(config, taskdesc, cache_type="docker-images.v1", cache_name=image_name, **kwargs) yield taskdesc
def fill_template(config, tasks): available_packages = {} for task in config.kind_dependencies_tasks: if task.kind != 'packages': continue name = task.label.replace('packages-', '') for route in task.task.get('routes', []): if route.startswith('index.') and '.hash.' in route: # Only keep the hash part of the route. h = route.rsplit('.', 1)[1] assert DIGEST_RE.match(h) available_packages[name] = h break context_hashes = {} for task in order_image_tasks(config, tasks): image_name = task.pop('name') job_symbol = task.pop('symbol') args = task.pop('args', {}) definition = task.pop('definition', image_name) packages = task.pop('packages', []) parent = task.pop('parent', None) for p in packages: if p not in available_packages: raise Exception('Missing package job for {}-{}: {}'.format( config.kind, image_name, p)) # Generating the context hash relies on arguments being set, so we # set this now, although it's not the final value (it's a # task-reference value, see further below). We add the package routes # containing a hash to get the overall docker image hash, so changes # to packages will be reflected in the docker image hash. args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p) for p in packages) if parent: args['DOCKER_IMAGE_PARENT'] = '{}:{}'.format(parent, context_hashes[parent]) context_path = os.path.join('taskcluster', 'docker', definition) context_hash = generate_context_hash( GECKO, context_path, image_name, args) digest_data = [context_hash] context_hashes[image_name] = context_hash description = 'Build the docker image {} for use by dependent tasks'.format( image_name) # Adjust the zstandard compression level based on the execution level. # We use faster compression for level 1 because we care more about # end-to-end times. We use slower/better compression for other levels # because images are read more often and it is worth the trade-off to # burn more CPU once to reduce image size. zstd_level = '3' if int(config.params['level']) == 1 else '10' # include some information that is useful in reconstructing this task # from JSON taskdesc = { 'label': 'build-docker-image-' + image_name, 'description': description, 'attributes': {'image_name': image_name}, 'expires-after': '28 days' if config.params.is_try() else '1 year', 'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'], 'treeherder': { 'symbol': job_symbol, 'platform': 'taskcluster-images/opt', 'kind': 'other', 'tier': 1, }, 'run-on-projects': [], 'worker-type': 'aws-provisioner-v1/gecko-{}-images'.format( config.params['level']), 'worker': { 'implementation': 'docker-worker', 'os': 'linux', 'artifacts': [{ 'type': 'file', 'path': '/builds/worker/workspace/artifacts/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { 'HG_STORE_PATH': '/builds/worker/checkouts/hg-store', 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], }, 'chain-of-trust': True, 'docker-in-docker': True, 'taskcluster-proxy': True, 'max-run-time': 7200, }, } worker = taskdesc['worker'] # We use the in-tree image_builder image to build docker images, but # that can't be used to build the image_builder image itself, # obviously. So we fall back to the last snapshot of the image that # was uploaded to docker hub. if image_name == 'image_builder': worker['docker-image'] = 'taskcluster/image_builder@sha256:' + \ '24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315' # Keep in sync with the Dockerfile used to generate the # docker image whose digest is referenced above. worker['volumes'] = [ '/builds/worker/checkouts', '/builds/worker/workspace', ] cache_name = 'imagebuilder-v1' else: worker['docker-image'] = {'in-tree': 'image_builder'} cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix()) # Force images built against the in-tree image builder to # have a different digest by adding a fixed string to the # hashed data. digest_data.append('image_builder') worker['caches'] = [{ 'type': 'persistent', 'name': 'level-{}-{}'.format(config.params['level'], cache_name), 'mount-point': '/builds/worker/checkouts', }] for k, v in args.items(): if k == 'DOCKER_IMAGE_PACKAGES': worker['env'][k] = {'task-reference': v} else: worker['env'][k] = v if packages: deps = taskdesc.setdefault('dependencies', {}) for p in sorted(packages): deps[p] = 'packages-{}'.format(p) digest_data.append(available_packages[p]) if parent: deps = taskdesc.setdefault('dependencies', {}) deps[parent] = 'build-docker-image-{}'.format(parent) worker['env']['DOCKER_IMAGE_PARENT_TASK'] = { 'task-reference': '<{}>'.format(parent) } if len(digest_data) > 1: kwargs = {'digest_data': digest_data} else: kwargs = {'digest': digest_data[0]} add_optimization( config, taskdesc, cache_type="docker-images.v1", cache_name=image_name, **kwargs ) yield taskdesc