def loader(kind, path, config, params, loaded_tasks):
# Build everything unless we have an optimization strategy (defined below).
files_changed = []
affected_components = ALL_COMPONENTS
if params["tasks_for"] == "github-pull-request":
logger.info("Processing pull request %s" %
params["pull_request_number"])
files_changed = get_files_changed_pr(params["base_repository"],
params["pull_request_number"])
affected_components = get_affected_components(
files_changed, config.get("files-affecting-components"))
elif params["tasks_for"] == "github-push":
if params["base_rev"] in _GIT_ZERO_HASHES:
logger.warn(
"base_rev is a zero hash, meaning there is no previous push. Building every component..."
)
else:
logger.info("Processing push for commit range %s -> %s" %
(params["base_rev"], params["head_rev"]))
files_changed = get_files_changed_push(params["base_repository"],
params["base_rev"],
params["head_rev"])
affected_components = get_affected_components(
files_changed, config.get("files-affecting-components"))
logger.info("Files changed: %s" % " ".join(files_changed))
if affected_components is ALL_COMPONENTS:
logger.info("Affected components: ALL")
else:
logger.info("Affected components: %s" % " ".join(affected_components))
not_for_components = config.get("not-for-components", [])
jobs = {
'{}{}'.format('' if build_type == 'regular' else build_type + '-',
component['name']): {
'attributes': {
'build-type': build_type,
'component': component['name'],
}
}
for component in get_components()
for build_type in ('regular', 'nightly', 'release')
if ((affected_components is ALL_COMPONENTS
or component['name'] in affected_components)
and component['name'] not in not_for_components and (
component['shouldPublish'] or build_type == 'regular'))
}
# Filter away overridden jobs that we wouldn't build anyways to avoid ending up with
# partial job entries.
overridden_jobs = {
k: v
for k, v in config.pop('overriden-jobs', {}).items()
if affected_components is ALL_COMPONENTS or k in jobs.keys()
}
jobs = merge(jobs, overridden_jobs)
config['jobs'] = jobs
return base_loader(kind, path, config, params, loaded_tasks)
def loader(kind, path, config, params, loaded_tasks):
not_for_locales = config.get("not-for-locales", [])
locales_per_chunk = config["locales-per-chunk"]
filtered_locales = [
locale for locale in get_screenshots_locales()
if locale not in not_for_locales
]
chunks, remainder = divmod(len(filtered_locales), locales_per_chunk)
if remainder:
# We need one last chunk to include locales in remainder
chunks = int(chunks + 1)
# Taskcluster sorts task names alphabetically, we need numbers to be zero-padded.
max_number_of_digits = _get_number_of_digits(chunks)
jobs = {
str(this_chunk).zfill(max_number_of_digits): {
"attributes": {
"chunk_locales": chunkify(filtered_locales, this_chunk,
chunks),
"l10n_chunk": str(this_chunk),
}
}
# Chunks starts at 1 (and not 0)
for this_chunk in range(1, chunks + 1)
}
config["jobs"] = jobs
return base_loader(kind, path, config, params, loaded_tasks)
def loader(kind, path, config, params, loaded_tasks):
config['jobs'] = jobs = {
component['name']: {
'attributes': {
'buildconfig': component
}
}
for component in get_components()
}
return base_loader(kind, path, config, params, loaded_tasks)
def loader(kind, path, config, params, loaded_tasks):
config['jobs'] = jobs = {
component['name']: {}
for component in get_components()
}
jobs.update(config.pop('overriden-jobs', {}))
for job_name, job_config in config.pop('additional-jobs', {}).iteritems():
if job_name in jobs:
raise ValueError('"{}" already exists in config["jobs"]'.format(job_name))
jobs[job_name] = job_config
return base_loader(kind, path, config, params, loaded_tasks)
def loader(kind, path, config, params, loaded_tasks):
not_for_components = config.get("not-for-components", [])
jobs = {
'{}{}'.format('' if build_type == 'regular' else build_type + '-',
component['name']): {
'attributes': {
'build-type': build_type,
'component': component['name'],
}
}
for component in get_components()
for build_type in ('regular', 'nightly', 'release', 'snapshot')
if (component['name'] not in not_for_components and (
component['shouldPublish'] or build_type == 'regular'))
}
jobs = merge(jobs, config.pop('overriden-jobs', {}))
config['jobs'] = jobs
return base_loader(kind, path, config, params, loaded_tasks)
def loader(kind, path, config, params, loaded_tasks):
# Build everything unless we have an optimization strategy (defined below).
files_changed = []
affected_components = ALL_COMPONENTS
upstream_component_dependencies = defaultdict(set)
downstream_component_dependencies = defaultdict(set)
for component, deps in get_upstream_deps_for_components(
[c["name"] for c in get_components()]):
if deps:
logger.info(
f"Found direct upstream dependencies for component '{component}': {deps}"
)
else:
logger.info(
"No direct upstream dependencies found for component '%s'" %
component)
upstream_component_dependencies[component] = deps
for d in deps:
downstream_component_dependencies[d].add(component)
if params["tasks_for"] == "github-pull-request":
logger.info("Processing pull request %s" %
params["pull_request_number"])
files_changed = get_files_changed_pr(params["base_repository"],
params["pull_request_number"])
affected_components = get_affected_components(
files_changed, config.get("files-affecting-components"),
upstream_component_dependencies, downstream_component_dependencies)
elif params["tasks_for"] == "github-push":
if params["base_rev"] in _GIT_ZERO_HASHES:
logger.warn(
"base_rev is a zero hash, meaning there is no previous push. Building every component..."
)
elif params["head_ref"] == "refs/heads/main":
# Disable the affected_components optimization to make sure we execute all tests to get
# a complete code coverage report for pushes to 'main'.
# See https://github.com/mozilla-mobile/android-components/issues/9382#issuecomment-760506327
logger.info(
"head_ref is refs/heads/main. Building every component...")
else:
logger.info("Processing push for commit range {} -> {}".format(
params["base_rev"], params["head_rev"]))
files_changed = get_files_changed_push(params["base_repository"],
params["base_rev"],
params["head_rev"])
affected_components = get_affected_components(
files_changed, config.get("files-affecting-components"),
upstream_component_dependencies,
downstream_component_dependencies)
logger.info("Files changed: %s" % " ".join(files_changed))
if affected_components is ALL_COMPONENTS:
logger.info("Affected components: ALL")
else:
logger.info("Affected components: %s" %
" ".join(sorted(affected_components)))
not_for_components = config.get("not-for-components", [])
jobs = {
'{}{}'.format('' if build_type == 'regular' else build_type + '-',
component['name']): {
'attributes': {
'build-type': build_type,
'component': component['name'],
}
}
for component in get_components()
for build_type in ('regular', 'nightly', 'release')
if ((affected_components is ALL_COMPONENTS
or component['name'] in affected_components)
and component['name'] not in not_for_components and (
component['shouldPublish'] or build_type == 'regular'))
}
# Filter away overridden jobs that we wouldn't build anyways to avoid ending up with
# partial job entries.
overridden_jobs = {
k: v
for k, v in config.pop('overriden-jobs', {}).items()
if affected_components is ALL_COMPONENTS or k in jobs.keys()
}
jobs = merge(jobs, overridden_jobs)
config['jobs'] = jobs
return base_loader(kind, path, config, params, loaded_tasks)
