def buildbot_graph_builder(builders, revision, complete=True):
""" Return graph of builders based on a list of builders.
# XXX: It would be better if had a BuildbotGraph class instead of messing
with dictionaries.
https://github.com/mozilla/mozilla_ci_tools/issues/353
Input: a list of builders and a revision
Output: a set which includes a graph with the builders we received, the necessary upstream
jobs and list of builders which can been used to schedule jobs through trigger_job()
NOTE: We will make it only return builder graph once Buildbot jobs are scheduled via
the Buildbot Bridge (BBB) instead of the normal Buildbot scheduling.
Graph of N levels:
{
'Builder a1': {
'Builder a2': {
...
'Builder aN': None
},
},
'Builder b1': None
}
:param builders: List of builder names
:type builders: list
:param revision: push revision
:type revision: str
:return: A graph of buildernames (single level of graphs)
:rtype: dict
:param complete: indicate that build has been completed or not
:type: booleans
"""
graph = {}
ready_to_trigger = []
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for b in builders:
if not valid_builder(buildername=b, quiet=True):
continue
if is_downstream(b):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective = determine_trigger_objective(revision, b)[0]
# The build job is already completed, we can trigger the test job
if objective == b:
# XXX: Fix me - Adding test jobs to the graph without a build associated
# to it does not work. This will be fixed once we switch to scheduling
# Buildbot jobs through BBB
if complete:
graph[b] = None
else:
ready_to_trigger.append(b)
# The build job is running, there is nothing we can do
elif objective is None:
pass
# We need to trigger the build job and the test job
else:
if objective not in graph:
graph[objective] = {}
graph[objective][b] = None
else:
if b not in graph:
graph[b] = {}
# We might have left a build job poiting to an empty dict
for builder in graph:
if graph[builder] == {}:
graph[builder] = None
return graph, ready_to_trigger
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
""" Return TC tasks based on a list of builders.
Input: a list of builders and a revision
Output: list of TC tasks base on builders we receive
:param builders: List of builder names
:type builders: list
:param repo_name: push revision
:type repo_name: str
:param revision: push revision
:type revision: str
:return: TC tasks
:rtype: dict
"""
tasks = []
build_builders = {}
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for builder in builders:
if is_upstream(builder):
properties = {'upload_to_task_id': slugId()}
# Bug 1274483 - Android multi-locale nightly builds need to upload to two different
# tasks, thus, it fails when we tell it to upload to the same task twice.
builder_details = get_buildername_metadata(builder)
if builder_details['platform_name'].startswith('android') and \
builder_details['nightly'] is True and \
'l10n' not in builder:
properties = {}
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties=properties,
)
tasks.append(task)
# We want to keep track of how many build builders we have
build_builders[builder] = task
for builder in builders:
if is_downstream(builder):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective, package_url, tests_url = \
determine_trigger_objective(revision, builder)
# The build job is already completed, we can trigger the test job
if objective == builder:
if objective in build_builders:
LOG.warning("We're creating a new build even though there's "
"already an existing completed build we could have "
"used. We hope you wanted to do this.")
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=build_builders[objective]['taskId'],
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
else:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
properties={
'packageUrl': package_url,
'testUrl': tests_url
},
)
tasks.append(task)
# The build job is running, there is nothing we can do
elif objective is None:
LOG.warning("We can add %s builder since the build associated "
"is running. This is because it is a Buildbot job.")
pass
# We need to trigger the build job and the test job
else:
if objective not in build_builders:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
taskId = task['taskId']
else:
taskId = build_builders[objective]['taskId']
# Add test job
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=taskId,
)
tasks.append(task)
return tasks
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
""" Return TC tasks based on a list of builders.
Input: a list of builders and a revision
Output: list of TC tasks base on builders we receive
:param builders: List of builder names
:type builders: list
:param repo_name: push revision
:type repo_name: str
:param revision: push revision
:type revision: str
:return: TC tasks
:rtype: dict
"""
tasks = []
build_builders = {}
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for builder in builders:
if is_upstream(builder):
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
# We want to keep track of how many build builders we have
build_builders[builder] = task
for builder in builders:
if is_downstream(builder):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective, package_url, tests_url = \
determine_trigger_objective(revision, builder)
# The build job is already completed, we can trigger the test job
if objective == builder:
if objective in build_builders:
LOG.warning(
"We're creating a new build even though there's "
"already an existing completed build we could have "
"used. We hope you wanted to do this.")
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=build_builders[objective]['taskId'],
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
else:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
properties={
'packageUrl': package_url,
'testUrl': tests_url
},
)
tasks.append(task)
# The build job is running, there is nothing we can do
elif objective is None:
LOG.warning(
"We can add %s builder since the build associated "
"is running. This is because it is a Buildbot job.")
pass
# We need to trigger the build job and the test job
else:
if objective not in build_builders:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties={'upload_to_task_id': slugId()},
)
tasks.append(task)
taskId = task['taskId']
else:
taskId = build_builders[objective]['taskId']
# Add test job
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=taskId,
)
tasks.append(task)
return tasks
def buildbot_graph_builder(builders, revision, complete=True):
""" Return graph of builders based on a list of builders.
# XXX: It would be better if had a BuildbotGraph class instead of messing
with dictionaries.
https://github.com/mozilla/mozilla_ci_tools/issues/353
Input: a list of builders and a revision
Output: a graph with the builders we received and the necessary upstream jobs
Graph of N levels:
{
'Builder a1': {
'Builder a2': {
...
'Builder aN': None
},
},
'Builder b1': None
}
:param builders: List of builder names
:type builders: list
:param revision: push revision
:type revision: str
:return: A graph of buildernames (single level of graphs)
:rtype: dict
"""
graph = {}
ready_to_trigger = []
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for b in builders:
if is_downstream(b):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective = determine_trigger_objective(revision, b)[0]
# The build job is already completed, we can trigger the test job
if objective == b:
# If complete is True, we add the job to our graph
if complete:
graph[b] = None
else:
ready_to_trigger.append(b)
# The build job is running, there is nothing we can do
elif objective is None:
pass
# We need to trigger the build job and the test job
else:
if objective not in graph:
graph[objective] = {}
graph[objective][b] = None
else:
if b not in graph:
graph[b] = {}
# We might have left a build job poiting to an empty dict
for builder in graph:
if graph[builder] == {}:
graph[builder] = None
return graph, ready_to_trigger
def buildbot_graph_builder(builders, revision):
""" Return graph of builders based on a list of builders.
# XXX: It would be better if had a BuildbotGraph class instead of messing
with dictionaries.
https://github.com/mozilla/mozilla_ci_tools/issues/353
Input: a list of builders and a revision
Output: a graph with the builders we received and the necessary upstream jobs
Graph of N levels:
{
'Builder a1': {
'Builder a2': {
...
'Builder aN': None
},
},
'Builder b1': None
}
:param builders: List of builder names
:type builders: list
:param revision: push revision
:type revision: str
:return: A graph of buildernames (single level of graphs)
:rtype: dict
"""
graph = {}
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for b in builders:
if is_downstream(b):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective = determine_trigger_objective(revision, b)[0]
# The build job is already completed, we can trigger the test job
if objective == b:
graph[b] = None
# The build job is running, there is nothing we can do
elif objective is None:
pass
# We need to trigger the build job and the test job
else:
if objective not in graph:
graph[objective] = {}
graph[objective][b] = None
else:
if b not in graph:
graph[b] = {}
# We might have left a build job poiting to an empty dict
for builder in graph:
if graph[builder] == {}:
graph[builder] = None
return graph
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
""" Return TC tasks based on a list of builders.
Input: a list of builders and a revision
Output: list of TC tasks base on builders we receive
:param builders: List of builder names
:type builders: list
:param repo_name: push revision
:type repo_name: str
:param revision: push revision
:type revision: str
:return: TC tasks
:rtype: dict
"""
tasks = []
build_builders = {}
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for builder in builders:
if is_upstream(builder):
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties={"upload_to_task_id": slugId()},
)
tasks.append(task)
# We want to keep track of how many build builders we have
build_builders[builder] = task
for builder in builders:
if is_downstream(builder):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective, package_url, tests_url = determine_trigger_objective(revision, builder)
# The build job is already completed, we can trigger the test job
if objective == builder:
if objective in build_builders:
LOG.warning(
"We're creating a new build even though there's "
"already an existing completed build we could have "
"used. We hope you wanted to do this."
)
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=build_builders[objective]["taskId"],
properties={"upload_to_task_id": slugId()},
)
tasks.append(task)
else:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
properties={"packageUrl": package_url, "testUrl": tests_url},
)
tasks.append(task)
# The build job is running, there is nothing we can do
elif objective is None:
LOG.warning(
"We can add %s builder since the build associated "
"is running. This is because it is a Buildbot job."
)
pass
# We need to trigger the build job and the test job
else:
if objective not in build_builders:
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
properties={"upload_to_task_id": slugId()},
)
tasks.append(task)
taskId = task["taskId"]
else:
taskId = build_builders[objective]["taskId"]
# Add test job
task = _create_task(
buildername=builder,
repo_name=repo_name,
revision=revision,
# task_graph_id=task_graph_id,
parent_task_id=taskId,
)
tasks.append(task)
return tasks
def buildbot_graph_builder(builders, revision, complete=True):
""" Return graph of builders based on a list of builders.
# XXX: It would be better if had a BuildbotGraph class instead of messing
with dictionaries.
https://github.com/mozilla/mozilla_ci_tools/issues/353
Input: a list of builders and a revision
Output: a set which includes a graph with the builders we received, the necessary upstream
jobs and list of builders which can been used to schedule jobs through trigger_job()
NOTE: We will make it only return builder graph once Buildbot jobs are scheduled via
the Buildbot Bridge (BBB) instead of the normal Buildbot scheduling.
Graph of N levels:
{
'Builder a1': {
'Builder a2': {
...
'Builder aN': None
},
},
'Builder b1': None
}
:param builders: List of builder names
:type builders: list
:param revision: push revision
:type revision: str
:return: A graph of buildernames (single level of graphs)
:rtype: dict
:param complete: indicate that build has been completed or not
:type: booleans
"""
graph = {}
ready_to_trigger = []
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for b in builders:
if not valid_builder(buildername=b, quiet=True):
continue
if is_downstream(b):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective = determine_trigger_objective(revision, b)[0]
# The build job is already completed, we can trigger the test job
if objective == b:
# XXX: Fix me - Adding test jobs to the graph without a build associated
# to it does not work. This will be fixed once we switch to scheduling
# Buildbot jobs through BBB
if complete:
graph[b] = None
else:
ready_to_trigger.append(b)
# The build job is running, there is nothing we can do
elif objective is None:
pass
# We need to trigger the build job and the test job
else:
if objective not in graph:
graph[objective] = {}
graph[objective][b] = None
else:
if b not in graph:
graph[b] = {}
# We might have left a build job poiting to an empty dict
for builder in graph:
if graph[builder] == {}:
graph[builder] = None
return graph, ready_to_trigger