def generate_tc_graph_from_builders(builders, repo_name, revision):
""" Return TC graph based on a list of builders.
:param builders: List of builder names
:type builders: list
:param repo_name: push revision
:type repo_name: str
:param revision: push revision
:type revision: str
:return: TC graph
:rtype: dict
"""
return generate_task_graph(
scopes=[
# This is needed to define tasks which take advantage of the BBB
'queue:define-task:buildbot-bridge/buildbot-bridge',
],
tasks=_generate_tc_tasks_from_builders(
builders=builders,
repo_name=repo_name,
revision=revision
),
metadata=generate_metadata(
repo_name=repo_name,
revision=revision,
name='Mozci BBB graph'
)
)
def test_metadata_contains_matches_name(self,
query_repo_url,
query_push_by_revision):
''' We want to test that the builder will show up in the name of the metadata.
This helps when we look at tasks when inspecting a graph scheduled via BBB.
See https://github.com/mozilla/mozilla_ci_tools/issues/444 for details.
'''
query_repo_url.return_value = self.repo_url
query_push_by_revision.return_value = self.push_info
builder = 'Platform1 try leak test build'
metadata = {
'name': builder,
'description': 'Task graph generated via Mozilla CI tools',
'owner': self.push_info.user,
'source': u'%s/rev/%s' % (self.repo_url, self.revision)
}
self.assertEquals(
generate_metadata(
repo_name=self.repo_name,
revision=self.revision,
name=builder
),
metadata
)
def trigger_builders_based_on_task_id(repo_name, revision, task_id, builders,
*args, **kwargs):
""" Create a graph of tasks which will use a TC task as their parent task.
:param repo_name The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: push revision
:type revision: str
:returns: Result of scheduling a TC graph
:rtype: dict
"""
if not builders:
return None
if type(builders) != list:
raise MozciError("builders must be a list")
# If the task_id is of a task which is running we want to extend the graph
# instead of submitting an independent one
task = get_task(task_id)
task_graph_id = task['taskGroupId']
state = get_task_graph_status(task_graph_id)
builders_graph, _ = buildbot_graph_builder(builders, revision)
if state == "running":
required_task_ids = [task_id]
else:
required_task_ids = []
task_graph = generate_task_graph(
scopes=[
# This is needed to define tasks which take advantage of the BBB
'queue:define-task:buildbot-bridge/buildbot-bridge',
],
tasks=_generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=builders_graph,
# This points to which parent to grab artifacts from
parent_task_id=task_id,
# This creates dependencies on other tasks
required_task_ids=required_task_ids,
),
metadata=generate_metadata(
repo_name=repo_name,
revision=revision,
name='Mozci BBB graph'
)
)
if state == "running":
result = extend_task_graph(task_graph_id, task_graph)
else:
result = schedule_graph(task_graph, *args, **kwargs)
LOG.info("Result from scheduling: %s" % result)
return result
def trigger_builders_based_on_task_id(repo_name, revision, task_id, builders,
*args, **kwargs):
""" Create a graph of tasks which will use a TC task as their parent task.
:param repo_name The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: push revision
:type revision: str
:returns: Result of scheduling a TC graph
:rtype: dict
"""
if not builders:
return None
if type(builders) != list:
raise MozciError("builders must be a list")
# If the task_id is of a task which is running we want to extend the graph
# instead of submitting an independent one
task = get_task(task_id)
task_graph_id = task['taskGroupId']
state = get_task_graph_status(task_graph_id)
builders_graph, _ = buildbot_graph_builder(builders, revision)
if state == "running":
required_task_ids = [task_id]
else:
required_task_ids = []
task_graph = generate_task_graph(
scopes=[
# This is needed to define tasks which take advantage of the BBB
'queue:define-task:buildbot-bridge/buildbot-bridge',
],
tasks=_generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=builders_graph,
# This points to which parent to grab artifacts from
parent_task_id=task_id,
# This creates dependencies on other tasks
required_task_ids=required_task_ids,
),
metadata=generate_metadata(repo_name=repo_name,
revision=revision,
name='Mozci BBB graph'))
if state == "running":
result = extend_task_graph(task_graph_id, task_graph)
else:
result = schedule_graph(task_graph, *args, **kwargs)
LOG.info("Result from scheduling: %s" % result)
return result
def generate_builders_tc_graph(repo_name, revision, builders_graph, *args, **kwargs):
"""Return TaskCluster graph based on builders_graph.
NOTE: We currently only support depending on one single parent.
:param repo_name The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: push revision
:type revision: str
:param builders_graph:
It is a graph made up of a dictionary where each
key is a Buildbot buildername. The value for each key is either None
or another graph of dependent builders.
:type builders_graph: dict
:returns: return None or a valid taskcluster task graph.
:rtype: dict
"""
LOG.debug("Generating TaskCluster BBB graph...")
if builders_graph is None:
return None
metadata = kwargs.get('metadata')
if metadata is None:
metadata = generate_metadata(repo_name=repo_name,
revision=revision,
name='Mozci BBB graph')
# This is the initial task graph which we're defining
task_graph = generate_task_graph(
scopes=[
# This is needed to define tasks which take advantage of the BBB
'queue:define-task:buildbot-bridge/buildbot-bridge',
],
tasks=_generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=builders_graph,
metadata=metadata,
),
metadata=metadata
)
return task_graph
def generate_builders_tc_graph(repo_name, revision, builders_graph, *args,
**kwargs):
"""Return TaskCluster graph based on builders_graph.
NOTE: We currently only support depending on one single parent.
:param repo_name The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: push revision
:type revision: str
:param builders_graph:
It is a graph made up of a dictionary where each
key is a Buildbot buildername. The value for each key is either None
or another graph of dependent builders.
:type builders_graph: dict
:returns: return None or a valid taskcluster task graph.
:rtype: dict
"""
LOG.debug("Generating TaskCluster BBB graph...")
if builders_graph is None:
return None
metadata = kwargs.get('metadata')
if metadata is None:
metadata = generate_metadata(repo_name=repo_name,
revision=revision,
name='Mozci BBB graph')
# This is the initial task graph which we're defining
task_graph = generate_task_graph(
scopes=[
# This is needed to define tasks which take advantage of the BBB
'queue:define-task:buildbot-bridge/buildbot-bridge',
],
tasks=_generate_tasks(
repo_name=repo_name,
revision=revision,
builders_graph=builders_graph,
metadata=metadata,
),
metadata=metadata)
return task_graph
def test_metadata_contains_matches_name(self, query_repo_url,
query_push_by_revision):
''' We want to test that the builder will show up in the name of the metadata.
This helps when we look at tasks when inspecting a graph scheduled via BBB.
See https://github.com/mozilla/mozilla_ci_tools/issues/444 for details.
'''
query_repo_url.return_value = self.repo_url
query_push_by_revision.return_value = self.push_info
builder = 'Platform1 mozilla-central leak test build'
metadata = {
'name': builder,
'description': 'Task graph generated via Mozilla CI tools',
'owner': self.push_info.user,
'source': u'%s/rev/%s' % (self.repo_url, self.revision)
}
self.assertEquals(
generate_metadata(repo_name=self.repo_name,
revision=self.revision,
name=builder), metadata)
def generate_tc_graph_from_builders(builders, repo_name, revision):
""" Return TC graph based on a list of builders.
:param builders: List of builder names
:type builders: list
:param repo_name: push revision
:type repo_name: str
:param revision: push revision
:type revision: str
:return: TC graph
:rtype: dict
"""
return generate_task_graph(
scopes=[
# This is needed to define tasks which take advantage of the BBB
'queue:define-task:buildbot-bridge/buildbot-bridge',
],
tasks=_generate_tc_tasks_from_builders(builders=builders,
repo_name=repo_name,
revision=revision),
metadata=generate_metadata(repo_name=repo_name,
revision=revision,
name='Mozci BBB graph'))
def _create_task(buildername, repo_name, revision, metadata=None, task_graph_id=None,
parent_task_id=None, requires=None, properties={}, *args, **kwargs):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param metadata: Metadata for the task. If not specified, generate it.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError(
"The builder '%s' should be for repo: %s." % (buildername, repo_name)
)
repo_url = query_repo_url(repo_name)
push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
full_revision = str(push_info.changesets[0].node)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info.user,
}
all_properties.update(properties)
all_properties.update(get_builder_extra_properties(buildername))
metadata = metadata if metadata is not None else \
generate_metadata(repo_name=repo_name,
revision=revision,
name=buildername)
# The task's name is used in the task-graph-inspector to list all tasks
# and using the buildername makes it easy for a person to recognize each job.
metadata['name'] = buildername
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': full_revision
},
'properties': all_properties,
},
metadata=metadata,
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties']['parent_task_id'] = parent_task_id
return task
def _create_task(buildername,
repo_name,
revision,
metadata=None,
task_graph_id=None,
parent_task_id=None,
requires=None,
properties={},
*args,
**kwargs):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param metadata: Metadata for the task. If not specified, generate it.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError("The builder '%s' should be for repo: %s." %
(buildername, repo_name))
repo_url = query_repo_url(repo_name)
push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
full_revision = str(push_info.changesets[0].node)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info.user,
}
all_properties.update(properties)
all_properties.update(get_builder_extra_properties(buildername))
metadata = metadata if metadata is not None else \
generate_metadata(repo_name=repo_name,
revision=revision,
name=buildername)
# The task's name is used in the task-graph-inspector to list all tasks
# and using the buildername makes it easy for a person to recognize each job.
metadata['name'] = buildername
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': full_revision
},
'properties': all_properties,
},
metadata=metadata,
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties'][
'parent_task_id'] = parent_task_id
return task