def _generate_tasks(repo_name,
revision,
builders_graph,
task_graph_id=None,
parent_task_id=None,
required_task_ids=[],
**kwargs):
""" Generate a TC json object with tasks based on a graph of graphs of buildernames
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param builders_graph:
It is a graph made up of a dictionary where each
key is a Buildbot buildername. The value for each key is either None
or another graph of dependent builders.
:type builders_graph: dict
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: int
:returns: A dictionary of TC tasks
:rtype: dict
"""
if not type(required_task_ids) == list:
raise MozciError("required_task_ids must be a list")
tasks = []
if type(builders_graph) != dict:
raise MozciError("The buildbot graph should be a dictionary")
# Let's iterate through the upstream builders
for builder, dependent_graph in builders_graph.iteritems():
task = _create_task(buildername=builder,
repo_name=repo_name,
revision=revision,
task_graph_id=task_graph_id,
parent_task_id=parent_task_id,
requires=required_task_ids,
**kwargs)
task_id = task['taskId']
tasks.append(task)
if dependent_graph:
# If there are builders this builder triggers let's add them as well
tasks = tasks + _generate_tasks(repo_name=repo_name,
revision=revision,
builders_graph=dependent_graph,
task_graph_id=task_graph_id,
required_task_ids=[task_id],
**kwargs)
return tasks
def _find_files(job_schedule_info):
"""
Find the files needed to trigger a job.
Raises MozciError if the job status doesn't have a properties key.
"""
files = {}
job_status = _status_info(job_schedule_info)
assert job_status is not None, \
"We should not have received an empty status"
properties = job_status.get("properties")
if not properties:
LOG.error(str(job_status))
raise MozciError("The status of the job is expected to have a "
"properties key, however, it is missing.")
LOG.debug("We want to find the files needed to trigger %s" %
properties["buildername"])
# We need the packageUrl, and one of testsUrl and testPackagesUrl,
# preferring testPackagesUrl.
if 'packageUrl' in properties:
files['packageUrl'] = properties['packageUrl']
if 'testPackagesUrl' in properties:
files['testPackagesUrl'] = properties['testPackagesUrl']
elif 'testsUrl' in properties:
files['testsUrl'] = properties['testsUrl']
return files
def trigger(builder,
revision,
files=None,
dry_run=False,
extra_properties=None):
"""Helper to trigger a job.
Returns a request.
"""
_add_builder_to_scheduling_manager(revision=revision, buildername=builder)
repo_name = query_repo_name_from_buildername(builder)
if is_downstream(builder) and not files:
raise MozciError(
'We have requested to trigger a test job, however, we have not provided '
'which files to run against.')
return trigger_arbitrary_job(repo_name=repo_name,
builder=builder,
revision=revision,
auth=get_credentials(),
files=files,
dry_run=dry_run,
extra_properties=extra_properties)
def trigger_talos_jobs_for_build(buildername,
revision,
times,
priority,
dry_run=False):
"""
Trigger all talos jobs for a given build and revision.
"""
failures = False
buildernames = get_talos_jobs_for_build(buildername)
for buildername in buildernames:
try:
trigger_range(buildername=buildername,
revisions=[revision],
times=times,
dry_run=dry_run)
except:
failures = True
LOG.warning('We failed to trigger {}; Let us try the rest.'.format(
buildername))
if failures:
raise MozciError(
"Some talos builders have failed to schedule; Check warning messages"
)
def trigger_builders_based_on_task_id(repo_name, revision, task_id, builders,
*args, **kwargs):
""" Create a graph of tasks which will use a TC task as their parent task.
:param repo_name The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: push revision
:type revision: str
:returns: Result of scheduling a TC graph
:rtype: dict
"""
if not builders:
return None
if type(builders) != list:
raise MozciError("builders must be a list")
# If the task_id is of a task which is running we want to extend the graph
# instead of submitting an independent one
task = get_task(task_id)
task_graph_id = task['taskGroupId']
state = get_task_graph_status(task_graph_id)
builders_graph = buildbot_graph_builder(builders, revision)
if state == "running":
required_task_ids = [task_id]
else:
required_task_ids = []
task_graph = generate_task_graph(
repo_name=repo_name,
revision=revision,
scopes=[
# This is needed to define tasks which take advantage of the BBB
'queue:define-task:buildbot-bridge/buildbot-bridge',
],
tasks=_generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=builders_graph,
# This points to which parent to grab artifacts from
parent_task_id=task_id,
# This creates dependencies on other tasks
required_task_ids=required_task_ids,
)
)
if state == "running":
result = extend_task_graph(task_graph_id, task_graph)
else:
result = schedule_graph(task_graph, *args, **kwargs)
print result
return result
def determine_upstream_builder(buildername):
"""
Given a builder name, find the build job that triggered it.
When buildername corresponds to a test job it determines the
triggering build job through allthethings.json. When a buildername
corresponds to a build job, it returns it unchanged.
Raises MozciError if no matching build job is found.
"""
_process_data()
# For some platforms in mozilla-beta and mozilla-aurora there are both
# talos and pgo talos jobs, only the pgo talos ones are valid.
if 'mozilla-beta' in buildername or 'mozilla-aurora' in buildername:
if 'talos' in buildername and 'pgo' not in buildername:
buildername_with_pgo = buildername.replace('talos', 'pgo talos')
if buildername_with_pgo.lower() in BUILDERNAME_TO_TRIGGER:
# The non pgo talos builders don't have a parent to trigger them
return
# If a buildername is in BUILD_JOBS, it means that it's a build job
# and it should be returned unchanged
if buildername.lower() in BUILD_JOBS:
return str(BUILD_JOBS[buildername.lower()])
# For some (but not all) platforms and repos, -pgo is explicit in
# the trigger but not in the shortname, e.g. "Linux
# mozilla-release build" shortname is "mozilla-release-linux" but
# the associated trigger name is
# "mozilla-release-linux-pgo-unittest"
SUFFIXES = ['-opt-unittest', '-unittest', '-talos', '-pgo']
# Guess the build job's shortname from the test job's trigger
# e.g. from "larch-android-api-11-opt-unittest"
# look for "larch-android-api-11" in SHORTNAME_TO_NAME and find
# "Android armv7 API 11+ larch build"
if buildername.lower() not in BUILDERNAME_TO_TRIGGER:
LOG.error("We didn't find a build job matching %s" % buildername)
raise MozciError("No build job matching %s found." % buildername)
shortname = BUILDERNAME_TO_TRIGGER[buildername.lower()]
for suffix in SUFFIXES:
if shortname.endswith(suffix):
shortname = shortname[:-len(suffix)]
if shortname in SHORTNAME_TO_NAME:
return str(SHORTNAME_TO_NAME[shortname])
# B2G jobs are weird
shortname = "b2g_" + shortname.replace('-emulator', '_emulator') + "_dep"
if shortname in SHORTNAME_TO_NAME:
return str(SHORTNAME_TO_NAME[shortname])
def query_repository(repo_name):
"""
Return dictionary with information about a specific repository.
Raises MozciError if the repository does not exist.
"""
repositories = query_repositories()
if repo_name not in repositories:
repositories = query_repositories(clobber=True)
if repo_name not in repositories:
raise MozciError("That repository does not exist.")
return repositories[repo_name]
def query_repo_name_from_buildername(buildername, clobber=False):
"""
Return the repository name from a given buildername.
Raises MozciError if there is no repository name in buildername.
"""
repositories_list = repositories.query_repositories(clobber)
ret_val = None
for repo_name in repositories_list:
if any(True for iterable in [' %s ', '_%s_', '-%s-']
if iterable % repo_name in buildername):
ret_val = repo_name
break
if ret_val is None and not clobber:
# Since repositories file is cached, it can be that something has changed.
# Adding clobber=True will make it overwrite the cached version with latest one.
query_repo_name_from_buildername(buildername, clobber=True)
if ret_val is None:
raise MozciError("Repository name not found in buildername. "
"Please provide a correct buildername.")
return ret_val
def determine_trigger_objective(revision,
buildername,
trigger_build_if_missing=True):
"""
Determine if we need to trigger any jobs and which job.
Returns:
* The name of the builder we need to trigger
* Files, if needed, to trigger such builder
"""
builder_to_trigger = None
files = None
repo_name = query_repo_name_from_buildername(buildername)
build_buildername = determine_upstream_builder(buildername)
if VALIDATE and not valid_builder(build_buildername):
raise MozciError("Our platforms mapping system has failed.")
if build_buildername == buildername:
# For a build job we know that we don't need files to
# trigger it and it's the build job we want to trigger
return build_buildername, None
# Let's figure out which jobs are associated to such revision
query_api = BuildApi()
# Let's only look at jobs that match such build_buildername
build_jobs = query_api.get_matching_jobs(repo_name, revision,
build_buildername)
# We need to determine if we need to trigger a build job
# or the test job
working_job = None
running_job = None
failed_job = None
LOG.debug("List of matching jobs:")
for job in build_jobs:
try:
status = query_api.get_job_status(job)
except buildjson.BuildjsonException:
LOG.debug(
"We have hit bug 1159279 and have to work around it. We will "
"pretend that we could not reach the files for it.")
continue
# Sometimes running jobs have status unknown in buildapi
if status in (RUNNING, PENDING, UNKNOWN):
LOG.debug(
"We found a running/pending build job. We don't search anymore."
)
running_job = job
# We cannot call _find_files for a running job
continue
# Having a coalesced build is the same as not having a build available
if status == COALESCED:
LOG.debug(
"The build we found was a coalesced one; this is the same as "
"non-existant.")
continue
# Successful or failed jobs may have the files we need
files = _find_files(job)
if files != [] and _all_urls_reachable(files.values()):
working_job = job
break
else:
LOG.debug("We can't determine the files for this build or "
"can't reach them.")
files = None
LOG.info("We found a job that finished but it did not "
"produced files. status: %d" % status)
failed_job = job
# End of for loop
if working_job:
# We found a build job with the necessary files. It could be a
# successful job, a running job that already emitted files or a
# testfailed job
LOG.debug(str(working_job))
LOG.info("We have the necessary files to trigger the downstream job.")
# We have the files needed to trigger the test job
builder_to_trigger = buildername
elif running_job:
LOG.info(
"We found a running/pending build job. We will not trigger another one."
)
LOG.info(
"You have to run the script again after the build job is finished to "
"trigger %s." % buildername)
builder_to_trigger = None
elif failed_job:
LOG.info(
"The build job %s failed on revision %s without generating the "
"necessary files. We will not trigger anything." %
(build_buildername, revision))
builder_to_trigger = None
else:
# We were trying to build a test job, however, we determined
# that we need an upstream builder instead
if not trigger_build_if_missing or not _unique_build_request(
build_buildername, revision):
# This is a safeguard to prevent triggering a build
# job multiple times if it is not intentional
builder_to_trigger = None
if not trigger_build_if_missing:
LOG.info(
"We would have to triggered build '%s' in order to trigger "
"job '%s'. On this mode we will not trigger either." %
(build_buildername, buildername))
else:
LOG.info("We will trigger 1) "
"'%s' instead of 2) '%s'" %
(build_buildername, buildername))
LOG.info("We need to trigger the build job once (1) "
"in order to be able to run the test job (2).")
if repo_name == 'try':
LOG.info(
"You'll need to run the script again after (1) is done to "
"trigger (2).")
else:
LOG.info(
"After (1) is done and if no coalesccing happens the test "
"jobs associated with it will be triggered.")
builder_to_trigger = build_buildername
if files:
return builder_to_trigger, files['packageUrl'], files[
'testPackagesUrl']
else:
return builder_to_trigger, None, None
def load_file(filename, url):
'''
We download a file without decompressing it so we can keep track of its progress.
We save it to disk and return the contents of it.
We also check if the file on the server is newer to determine if we should download it again.
Raises MozciError if anything goes wrong.
'''
# Obtain the absolute path to our file in the cache
if not os.path.isabs(filename):
filepath = path_to_file(filename)
else:
filepath = filename
headers = {
'Accept-Encoding': None,
}
exists = os.path.exists(filepath)
if exists:
# The file exists in the cache, let's verify that is still current
statinfo = os.stat(filepath)
last_mod_date = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
time.gmtime(statinfo.st_mtime))
headers['If-Modified-Since'] = last_mod_date
else:
# The file does not exist in the cache; let's fetch
LOG.debug("We have not been able to find %s on disk." % filepath)
req = requests.get(url, stream=True, headers=headers)
if req.status_code == 200:
if exists:
# The file on the server is newer
LOG.info("The local file was last modified in %s." % last_mod_date)
LOG.info("The server's last modified in %s" %
req.headers['last-modified'])
LOG.info("We need to fetch it again.")
_save_file(req, filename)
elif req.status_code == 304:
# The file on disk is recent
LOG.debug("%s is on disk and it is current." % last_mod_date)
else:
raise MozciError("We received %s which is unexpected." %
req.status_code)
try:
if not MEMORY_SAVING_MODE:
return _load_json_file(filepath)
return _lean_load_json_file(filepath)
# Issue 213: sometimes we download a corrupted builds-*.js file
except (IOError, subprocess.CalledProcessError):
LOG.info("%s is corrupted, we will have to download a new one.",
filename)
os.remove(filepath)
return load_file(filename, url)
def _create_task(buildername,
repo_name,
revision,
task_graph_id=None,
parent_task_id=None,
requires=None,
properties={}):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError("The builder '%s' should be for repo: %s." %
(buildername, repo_name))
repo_url = query_repo_url(repo_name)
push_info = query_revision_info(repo_url, revision)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info['user'],
}
all_properties.update(properties)
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': revision
},
'properties': all_properties,
},
metadata_name=buildername)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties'][
'parent_task_id'] = parent_task_id
return task
def _generate_tasks(repo_name,
revision,
builders_graph,
metadata=None,
task_graph_id=None,
parent_task_id=None,
required_task_ids=[],
**kwargs):
""" Generate a TC json object with tasks based on a graph of graphs of buildernames
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param builders_graph:
It is a graph made up of a dictionary where each
key is a Buildbot buildername. The value for each key is either None
or another graph of dependent builders.
:type builders_graph: dict
:param metadata: Metadata information to set for the tasks.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: int
:returns: A dictionary of TC tasks
:rtype: dict
"""
if not type(required_task_ids) == list:
raise MozciError("required_task_ids must be a list")
tasks = []
if type(builders_graph) != dict:
raise MozciError("The buildbot graph should be a dictionary")
# Let's iterate through the root builders in this graph
for builder, dependent_graph in builders_graph.iteritems():
# Due to bug 1221091 this will be used to know to which task
# the artifacts will be uploaded to
upload_to_task_id = slugId()
properties = {'upload_to_task_id': upload_to_task_id}
builder_details = get_buildername_metadata(builder)
# Bug 1274483 - Android multi-locale nightly builds need to upload to two different tasks,
# thus, it fails when we tell it to upload to the same task twice.
if builder_details['platform_name'].startswith('android') and \
builder_details['nightly'] is True and \
'l10n' not in builder:
properties = {}
task = _create_task(buildername=builder,
repo_name=repo_name,
revision=revision,
metadata=metadata,
task_graph_id=task_graph_id,
parent_task_id=parent_task_id,
properties=properties,
requires=required_task_ids,
**kwargs)
task_id = task['taskId']
tasks.append(task)
if dependent_graph:
# If there are builders this builder triggers let's add them as well
tasks = tasks + _generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=dependent_graph,
metadata=metadata,
task_graph_id=task_graph_id,
# The parent task id is used to find artifacts; only one can be given
parent_task_id=upload_to_task_id,
# The required tasks are the one holding this task from running
required_task_ids=[task_id],
**kwargs)
return tasks
def _create_task(buildername,
repo_name,
revision,
metadata=None,
task_graph_id=None,
parent_task_id=None,
requires=None,
properties={},
*args,
**kwargs):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param metadata: Metadata for the task. If not specified, generate it.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError("The builder '%s' should be for repo: %s." %
(buildername, repo_name))
repo_url = query_repo_url(repo_name)
push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
full_revision = str(push_info.changesets[0].node)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info.user,
}
all_properties.update(properties)
all_properties.update(get_builder_extra_properties(buildername))
metadata = metadata if metadata is not None else \
generate_metadata(repo_name=repo_name,
revision=revision,
name=buildername)
# The task's name is used in the task-graph-inspector to list all tasks
# and using the buildername makes it easy for a person to recognize each job.
metadata['name'] = buildername
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': full_revision
},
'properties': all_properties,
},
metadata=metadata,
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties'][
'parent_task_id'] = parent_task_id
return task