def get_digest_data(config, run, taskdesc): files = list(run.get('resources', [])) # This file files.append('taskcluster/taskgraph/transforms/job/toolchain.py') # The script files.append('taskcluster/scripts/misc/{}'.format(run['script'])) # Tooltool manifest if any is defined: tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST') if tooltool_manifest: files.append(tooltool_manifest) # Accumulate dependency hashes for index generation. data = [hash_paths(GECKO, files)] # If the task has dependencies, we need those dependencies to influence # the index path. So take the digest from the files above, add the list # of its dependencies, and hash the aggregate. # If the task has no dependencies, just use the digest from above. deps = taskdesc['dependencies'] if deps: data.extend(sorted(deps.values())) # Likewise script arguments should influence the index. args = run.get('arguments') if args: data.extend(args) return data
def get_digest_data(config, run, taskdesc): files = list(run.pop('resources', [])) # The script files.append('taskcluster/scripts/misc/{}'.format(run['script'])) # Tooltool manifest if any is defined: tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST') if tooltool_manifest: files.append(tooltool_manifest) # Accumulate dependency hashes for index generation. data = [hash_paths(GECKO, files)] data.append(taskdesc['attributes']['toolchain-artifact']) # If the task uses an in-tree docker image, we want it to influence # the index path as well. Ideally, the content of the docker image itself # should have an influence, but at the moment, we can't get that # information here. So use the docker image name as a proxy. Not a lot of # changes to docker images actually have an impact on the resulting # toolchain artifact, so we'll just rely on such important changes to be # accompanied with a docker image name change. image = taskdesc['worker'].get('docker-image', {}).get('in-tree') if image: data.append(image) # Likewise script arguments should influence the index. args = run.get('arguments') if args: data.extend(args) if taskdesc['attributes'].get('rebuild-on-release'): # Add whether this is a release branch or not data.append(str(config.params['project'] in RELEASE_PROJECTS)) return data
def build_cache(config, tasks): for task in tasks: # Only cache tasks on PRs and push. Ignore actions. if config.params["tasks_for"] not in ("github-pull-request", "github-push"): yield task continue if task.get("cache", True) and not taskgraph.fast: digest_data = [] digest_data.append( json.dumps( task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True, ) ) resources = task["attributes"]["resources"] for resource in resources: path = os.path.join(BASE_DIR, resource) if os.path.isdir(resource): digest_data.append(hash_paths(path, [""])) elif os.path.isfile(resource): digest_data.append(hash_path(path)) else: raise Exception(f"Unknown resource {resource}") cache_name = task["name"].replace(":", "-") task["cache"] = { "type": f"xpi-manifest.v1.{config.kind}", "name": cache_name, "digest-data": digest_data, } yield task
def get_digest_data(config, run, taskdesc): """ Copied from taskgraph.transforms.job.toolchain, with minor modifications to support the required script path. """ files = list(run.pop('resources', [])) # This file files.append('comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py') # The script files.append('{}/{}'.format(TOOLCHAIN_SCRIPT_PATH, run['script'])) # Tooltool manifest if any is defined: tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST') if tooltool_manifest: files.append(tooltool_manifest) # Accumulate dependency hashes for index generation. data = [hash_paths(GECKO, files)] # If the task uses an in-tree docker image, we want it to influence # the index path as well. Ideally, the content of the docker image itself # should have an influence, but at the moment, we can't get that # information here. So use the docker image name as a proxy. Not a lot of # changes to docker images actually have an impact on the resulting # toolchain artifact, so we'll just rely on such important changes to be # accompanied with a docker image name change. image = taskdesc['worker'].get('docker-image', {}).get('in-tree') if image: data.append(image) # Likewise script arguments should influence the index. args = run.get('arguments') if args: data.extend(args) return data
def add_optimization(config, run, taskdesc): files = list(run.get('resources', [])) # This file files.append('taskcluster/taskgraph/transforms/job/toolchain.py') # The script files.append('taskcluster/scripts/misc/{}'.format(run['script'])) # Tooltool manifest if any is defined: tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST') if tooltool_manifest: files.append(tooltool_manifest) # Accumulate dependency hashes for index generation. data = [hash_paths(GECKO, files)] # If the task has dependencies, we need those dependencies to influence # the index path. So take the digest from the files above, add the list # of its dependencies, and hash the aggregate. # If the task has no dependencies, just use the digest from above. deps = taskdesc['dependencies'] if deps: data.extend(sorted(deps.values())) # Likewise script arguments should influence the index. args = run.get('arguments') if args: data.extend(args) label = taskdesc['label'] subs = { 'name': label.replace('%s-' % config.kind, ''), 'digest': hashlib.sha256('\n'.join(data)).hexdigest() } # We'll try to find a cached version of the toolchain at levels above # and including the current level, starting at the highest level. index_routes = [] for level in reversed(range(int(config.params['level']), 4)): subs['level'] = level index_routes.append(TOOLCHAIN_INDEX.format(**subs)) taskdesc['optimization'] = {'index-search': index_routes} # ... and cache at the lowest level. taskdesc.setdefault('routes', []).append('index.{}'.format( TOOLCHAIN_INDEX.format(**subs)))
def build_cache(config, tasks): for task in tasks: if task.get("cache", True) and not taskgraph.fast: digest_data = [] digest_data.append( json.dumps(task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True)) resources = task["attributes"]["resources"] for resource in resources: digest_data.append( hash_paths(os.path.join(BASE_DIR, resource), [''])) cache_name = task["name"].replace(":", "-") task["cache"] = { "type": "scriptworker-scripts.v1.{}".format(config.kind), "name": cache_name, "digest-data": digest_data, } yield task
def get_digest_data(config, run, taskdesc): files = list(run.get('resources', [])) # This file files.append('taskcluster/taskgraph/transforms/job/toolchain.py') # The script files.append('taskcluster/scripts/misc/{}'.format(run['script'])) # Tooltool manifest if any is defined: tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST') if tooltool_manifest: files.append(tooltool_manifest) # Accumulate dependency hashes for index generation. data = [hash_paths(GECKO, files)] # If the task has dependencies, we need those dependencies to influence # the index path. So take the digest from the files above, add the list # of its dependencies, and hash the aggregate. # If the task has no dependencies, just use the digest from above. deps = taskdesc['dependencies'] if deps: data.extend(sorted(deps.values())) # If the task uses an in-tree docker image, we want it to influence # the index path as well. Ideally, the content of the docker image itself # should have an influence, but at the moment, we can't get that # information here. So use the docker image name as a proxy. Not a lot of # changes to docker images actually have an impact on the resulting # toolchain artifact, so we'll just rely on such important changes to be # accompanied with a docker image name change. image = taskdesc['worker'].get('docker-image', {}).get('in-tree') if image: data.extend(image) # Likewise script arguments should influence the index. args = run.get('arguments') if args: data.extend(args) return data
def get_digest_data(config, run, taskdesc): files = list(run.get('resources', [])) # This file files.append('taskcluster/taskgraph/transforms/job/toolchain.py') # The script files.append('taskcluster/scripts/misc/{}'.format(run['script'])) # Tooltool manifest if any is defined: tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST') if tooltool_manifest: files.append(tooltool_manifest) # Accumulate dependency hashes for index generation. data = [hash_paths(GECKO, files)] # If the task has dependencies, we need those dependencies to influence # the index path. So take the digest from the files above, add the list # of its dependencies, and hash the aggregate. # If the task has no dependencies, just use the digest from above. deps = taskdesc['dependencies'] if deps: data.extend(sorted(deps.values())) # If the task uses an in-tree docker image, we want it to influence # the index path as well. Ideally, the content of the docker image itself # should have an influence, but at the moment, we can't get that # information here. So use the docker image name as a proxy. Not a lot of # changes to docker images actually have an impact on the resulting # toolchain artifact, so we'll just rely on such important changes to be # accompanied with a docker image name change. image = taskdesc['worker'].get('docker-image', {}).get('in-tree') if image: data.extend(image) # Likewise script arguments should influence the index. args = run.get('arguments') if args: data.extend(args) return data
def add_optimizations(config, run, taskdesc): files = list(run.get('resources', [])) # This file files.append('taskcluster/taskgraph/transforms/job/toolchain.py') # The script files.append('taskcluster/scripts/misc/{}'.format(run['script'])) label = taskdesc['label'] subs = { 'name': label.replace('toolchain-', '').split('/')[0], 'digest': hash_paths(GECKO, files), } optimizations = taskdesc.setdefault('optimizations', []) # We'll try to find a cached version of the toolchain at levels above # and including the current level, starting at the highest level. for level in reversed(range(int(config.params['level']), 4)): subs['level'] = level optimizations.append(['index-search', TOOLCHAIN_INDEX.format(**subs)]) # ... and cache at the lowest level. taskdesc.setdefault('routes', []).append( 'index.{}'.format(TOOLCHAIN_INDEX.format(**subs)))
def add_optimizations(config, run, taskdesc): files = list(run.get('resources', [])) # This file files.append('taskcluster/taskgraph/transforms/job/toolchain.py') # The script files.append('taskcluster/scripts/misc/{}'.format(run['script'])) label = taskdesc['label'] subs = { 'name': label.replace('toolchain-', '').split('/')[0], 'digest': hash_paths(GECKO, files), } optimizations = taskdesc.setdefault('optimizations', []) # We'll try to find a cached version of the toolchain at levels above # and including the current level, starting at the highest level. for level in reversed(range(int(config.params['level']), 4)): subs['level'] = level optimizations.append(['index-search', TOOLCHAIN_INDEX.format(**subs)]) # ... and cache at the lowest level. taskdesc.setdefault('routes', []).append( 'index.{}'.format(TOOLCHAIN_INDEX.format(**subs)))
def get_digest_data(config, run, taskdesc): files = list(run.pop("resources", [])) # The script files.append("taskcluster/scripts/toolchain/{}".format(run["script"])) # Accumulate dependency hashes for index generation. data = [hash_paths(config.graph_config.vcs_root, files)] # If the task uses an in-tree docker image, we want it to influence # the index path as well. Ideally, the content of the docker image itself # should have an influence, but at the moment, we can't get that # information here. So use the docker image name as a proxy. Not a lot of # changes to docker images actually have an impact on the resulting # toolchain artifact, so we'll just rely on such important changes to be # accompanied with a docker image name change. image = taskdesc["worker"].get("docker-image", {}).get("in-tree") if image: data.append(image) # Likewise script arguments should influence the index. args = run.get("arguments") if args: data.extend(args) return data