def test_nightly_build(test_job, expected):
"""Test whether nightly builds are caught by get_buildername_metadata"""
import mozci.platforms
mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
obtained = get_buildername_metadata(test_job)['nightly']
assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained,
expected)
def test_suite_name(test_job, expected):
"""Test if the function finds the right suite_name."""
import mozci.platforms
mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
obtained = get_buildername_metadata(test_job)['suite_name']
assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained,
expected)
def test_suite_name(self, fetch_allthethings_data):
"""Test if the function finds the right suite_name."""
fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS
test_cases = [("Platform1 repo talos tp5o", "tp5o"),
("Platform1 repo opt test mochitest-1", "mochitest-1")]
for t in test_cases:
self.assertEquals(
get_buildername_metadata(t[0])['suite_name'], t[1])
def test_suite_name(self, fetch_allthethings_data):
"""Test if the function finds the right suite_name."""
fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS
test_cases = [
("Platform1 repo talos tp5o", "tp5o"),
("Platform1 repo opt test mochitest-1", "mochitest-1")
]
for t in test_cases:
self.assertEquals(get_buildername_metadata(t[0])['suite_name'], t[1])
def ignored(data):
'''It determines if the request will be processed or not.'''
try:
info = get_buildername_metadata(data['payload']['buildername'])
if info['build_type'] == "pgo" and \
info['repo_name'] in ('mozilla-inbound', 'fx-team', 'autoland') and \
info['platform_name'] != 'win64':
return False
else:
return True
except MissingBuilderError, e:
LOG.warning(str(e))
return True
def ignored(data):
"""It determines if the request will be processed or not."""
try:
info = get_buildername_metadata(data["payload"]["buildername"])
if (
info["build_type"] == "pgo"
and info["repo_name"] in ("mozilla-inbound", "fx-team", "autoland")
and info["platform_name"] != "win64"
):
return False
else:
return True
except MissingBuilderError, e:
LOG.warning(str(e))
return True
def _generate_tasks(repo_name, revision, builders_graph, metadata=None, task_graph_id=None,
parent_task_id=None, required_task_ids=[], **kwargs):
""" Generate a TC json object with tasks based on a graph of graphs of buildernames
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param builders_graph:
It is a graph made up of a dictionary where each
key is a Buildbot buildername. The value for each key is either None
or another graph of dependent builders.
:type builders_graph: dict
:param metadata: Metadata information to set for the tasks.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: int
:returns: A dictionary of TC tasks
:rtype: dict
"""
if not type(required_task_ids) == list:
raise MozciError("required_task_ids must be a list")
tasks = []
if type(builders_graph) != dict:
raise MozciError("The buildbot graph should be a dictionary")
# Let's iterate through the root builders in this graph
for builder, dependent_graph in builders_graph.iteritems():
# Due to bug 1221091 this will be used to know to which task
# the artifacts will be uploaded to
upload_to_task_id = slugId()
properties = {'upload_to_task_id': upload_to_task_id}
builder_details = get_buildername_metadata(builder)
# Bug 1274483 - Android multi-locale nightly builds need to upload to two different tasks,
# thus, it fails when we tell it to upload to the same task twice.
if builder_details['platform_name'].startswith('android') and \
builder_details['nightly'] is True and \
'l10n' not in builder:
properties = {}
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
metadata=metadata,
task_graph_id=task_graph_id,
parent_task_id=parent_task_id,
properties=properties,
requires=required_task_ids,
**kwargs
)
task_id = task['taskId']
tasks.append(task)
if dependent_graph:
# If there are builders this builder triggers let's add them as well
tasks = tasks + _generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=dependent_graph,
metadata=metadata,
task_graph_id=task_graph_id,
# The parent task id is used to find artifacts; only one can be given
parent_task_id=upload_to_task_id,
# The required tasks are the one holding this task from running
required_task_ids=[task_id],
**kwargs
)
return tasks
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
""" Return TC tasks based on a list of builders.
Input: a list of builders and a revision
Output: list of TC tasks base on builders we receive
:param builders: List of builder names
:type builders: list
:param repo_name: push revision
:type repo_name: str
:param revision: push revision
:type revision: str
:return: TC tasks
:rtype: dict
"""
tasks = []
build_builders = {}
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for builder in builders:
if is_upstream(builder):
properties = {'upload_to_task_id': slugId()}
# Bug 1274483 - Android multi-locale nightly builds need to upload to two different
# tasks, thus, it fails when we tell it to upload to the same task twice.
builder_details = get_buildername_metadata(builder)
if builder_details['platform_name'].startswith('android') and \
builder_details['nightly'] is True and \
'l10n' not in builder:
properties = {}
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties=properties,
)
tasks.append(task)
# We want to keep track of how many build builders we have
build_builders[builder] = task
for builder in builders:
if is_downstream(builder):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective, package_url, tests_url = \
determine_trigger_objective(revision, builder)
# The build job is already completed, we can trigger the test job
if objective == builder:
if objective in build_builders:
LOG.warning("We're creating a new build even though there's "
"already an existing completed build we could have "
"used. We hope you wanted to do this.")
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=build_builders[objective]['taskId'],
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
else:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
properties={
'packageUrl': package_url,
'testUrl': tests_url
},
)
tasks.append(task)
# The build job is running, there is nothing we can do
elif objective is None:
LOG.warning("We can add %s builder since the build associated "
"is running. This is because it is a Buildbot job.")
pass
# We need to trigger the build job and the test job
else:
if objective not in build_builders:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
taskId = task['taskId']
else:
taskId = build_builders[objective]['taskId']
# Add test job
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=taskId,
)
tasks.append(task)
return tasks
def _create_task(buildername, repo_name, revision, metadata=None, task_graph_id=None,
parent_task_id=None, requires=None, properties={}, *args, **kwargs):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param metadata: Metadata for the task. If not specified, generate it.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError(
"The builder '%s' should be for repo: %s." % (buildername, repo_name)
)
repo_url = query_repo_url(repo_name)
push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
full_revision = str(push_info.changesets[0].node)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info.user,
}
all_properties.update(properties)
all_properties.update(get_builder_extra_properties(buildername))
metadata = metadata if metadata is not None else \
generate_metadata(repo_name=repo_name,
revision=revision,
name=buildername)
# The task's name is used in the task-graph-inspector to list all tasks
# and using the buildername makes it easy for a person to recognize each job.
metadata['name'] = buildername
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': full_revision
},
'properties': all_properties,
},
metadata=metadata,
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties']['parent_task_id'] = parent_task_id
return task
def _create_task(buildername,
repo_name,
revision,
task_graph_id=None,
parent_task_id=None,
requires=None,
properties={}):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError("The builder '%s' should be for repo: %s." %
(buildername, repo_name))
repo_url = query_repo_url(repo_name)
push_info = query_revision_info(repo_url, revision)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info['user'],
}
all_properties.update(properties)
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': revision
},
'properties': all_properties,
},
metadata_name=buildername)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties'][
'parent_task_id'] = parent_task_id
return task
def _create_task(buildername, repo_name, revision, task_graph_id=None,
parent_task_id=None, requires=None, properties={}):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError(
"The builder '%s' should be for repo: %s." % (buildername, repo_name)
)
repo_url = query_repo_url(repo_name)
push_info = query_revision_info(repo_url, revision)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info['user'],
}
all_properties.update(properties)
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': revision
},
'properties': all_properties,
},
metadata_name=buildername
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties']['parent_task_id'] = parent_task_id
return task
def test_nightly_build(test_job, expected):
"""Test whether nightly builds are caught by get_buildername_metadata"""
import mozci.platforms
mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
obtained = get_buildername_metadata(test_job)['nightly']
assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained, expected)
def test_suite_name(test_job, expected):
"""Test if the function finds the right suite_name."""
import mozci.platforms
mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
obtained = get_buildername_metadata(test_job)['suite_name']
assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained, expected)
def _generate_tasks(repo_name,
revision,
builders_graph,
metadata=None,
task_graph_id=None,
parent_task_id=None,
required_task_ids=[],
**kwargs):
""" Generate a TC json object with tasks based on a graph of graphs of buildernames
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param builders_graph:
It is a graph made up of a dictionary where each
key is a Buildbot buildername. The value for each key is either None
or another graph of dependent builders.
:type builders_graph: dict
:param metadata: Metadata information to set for the tasks.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: int
:returns: A dictionary of TC tasks
:rtype: dict
"""
if not type(required_task_ids) == list:
raise MozciError("required_task_ids must be a list")
tasks = []
if type(builders_graph) != dict:
raise MozciError("The buildbot graph should be a dictionary")
# Let's iterate through the root builders in this graph
for builder, dependent_graph in builders_graph.iteritems():
# Due to bug 1221091 this will be used to know to which task
# the artifacts will be uploaded to
upload_to_task_id = slugId()
properties = {'upload_to_task_id': upload_to_task_id}
builder_details = get_buildername_metadata(builder)
# Bug 1274483 - Android multi-locale nightly builds need to upload to two different tasks,
# thus, it fails when we tell it to upload to the same task twice.
if builder_details['platform_name'].startswith('android') and \
builder_details['nightly'] is True and \
'l10n' not in builder:
properties = {}
task = _create_task(buildername=builder,
repo_name=repo_name,
revision=revision,
metadata=metadata,
task_graph_id=task_graph_id,
parent_task_id=parent_task_id,
properties=properties,
requires=required_task_ids,
**kwargs)
task_id = task['taskId']
tasks.append(task)
if dependent_graph:
# If there are builders this builder triggers let's add them as well
tasks = tasks + _generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=dependent_graph,
metadata=metadata,
task_graph_id=task_graph_id,
# The parent task id is used to find artifacts; only one can be given
parent_task_id=upload_to_task_id,
# The required tasks are the one holding this task from running
required_task_ids=[task_id],
**kwargs)
return tasks
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
""" Return TC tasks based on a list of builders.
Input: a list of builders and a revision
Output: list of TC tasks base on builders we receive
:param builders: List of builder names
:type builders: list
:param repo_name: push revision
:type repo_name: str
:param revision: push revision
:type revision: str
:return: TC tasks
:rtype: dict
"""
tasks = []
build_builders = {}
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for builder in builders:
if is_upstream(builder):
properties = {'upload_to_task_id': slugId()}
# Bug 1274483 - Android multi-locale nightly builds need to upload to two different
# tasks, thus, it fails when we tell it to upload to the same task twice.
builder_details = get_buildername_metadata(builder)
if builder_details['platform_name'].startswith('android') and \
builder_details['nightly'] is True and \
'l10n' not in builder:
properties = {}
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties=properties,
)
tasks.append(task)
# We want to keep track of how many build builders we have
build_builders[builder] = task
for builder in builders:
if is_downstream(builder):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective, package_url, tests_url = \
determine_trigger_objective(revision, builder)
# The build job is already completed, we can trigger the test job
if objective == builder:
if objective in build_builders:
LOG.warning(
"We're creating a new build even though there's "
"already an existing completed build we could have "
"used. We hope you wanted to do this.")
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=build_builders[objective]['taskId'],
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
else:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
properties={
'packageUrl': package_url,
'testUrl': tests_url
},
)
tasks.append(task)
# The build job is running, there is nothing we can do
elif objective is None:
LOG.warning(
"We can add %s builder since the build associated "
"is running. This is because it is a Buildbot job.")
pass
# We need to trigger the build job and the test job
else:
if objective not in build_builders:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
taskId = task['taskId']
else:
taskId = build_builders[objective]['taskId']
# Add test job
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=taskId,
)
tasks.append(task)
return tasks
def _create_task(buildername,
repo_name,
revision,
metadata=None,
task_graph_id=None,
parent_task_id=None,
requires=None,
properties={},
*args,
**kwargs):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param metadata: Metadata for the task. If not specified, generate it.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError("The builder '%s' should be for repo: %s." %
(buildername, repo_name))
repo_url = query_repo_url(repo_name)
push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
full_revision = str(push_info.changesets[0].node)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info.user,
}
all_properties.update(properties)
all_properties.update(get_builder_extra_properties(buildername))
metadata = metadata if metadata is not None else \
generate_metadata(repo_name=repo_name,
revision=revision,
name=buildername)
# The task's name is used in the task-graph-inspector to list all tasks
# and using the buildername makes it easy for a person to recognize each job.
metadata['name'] = buildername
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': full_revision
},
'properties': all_properties,
},
metadata=metadata,
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties'][
'parent_task_id'] = parent_task_id
return task