def _possibly_valid_builder(buildername):
'''Return an equivalent valid builder if it can be found, otherwise, return None.
Treeherder is sending us old buildernames and in some cases we can return the valid
builder. We also safe guard when the substitution does not work or needs refreshing.
'''
if valid_builder(buildername):
return buildername
new_builder = None
# We might be able to replace the builder for the right one
# Bug 1242038 - Treeherder sends the wrong buildernames
for repl in BUILDERNAME_REPLACEMENTS:
if buildername.startswith(repl[0]):
new_builder = buildername.replace(repl[0], repl[1])
LOG.warning('Old builder: %s New builder: %s' %
(buildername, new_builder))
break
# We either did not replace or the replacement is also invalid
if new_builder is not None and not valid_builder(new_builder):
new_builder = None
return new_builder
def _possibly_valid_builder(buildername):
'''Return an equivalent valid builder if it can be found, otherwise, return None.
Treeherder is sending us old buildernames and in some cases we can return the valid
builder. We also safe guard when the substitution does not work or needs refreshing.
'''
if valid_builder(buildername):
return buildername
new_builder = None
# We might be able to replace the builder for the right one
# Bug 1242038 - Treeherder sends the wrong buildernames
for repl in BUILDERNAME_REPLACEMENTS:
if buildername.startswith(repl[0]):
new_builder = buildername.replace(repl[0], repl[1])
LOG.warning('Old builder: %s New builder: %s' %
(buildername, new_builder))
break
# We either did not replace or the replacement is also invalid
if new_builder is not None and not valid_builder(new_builder):
new_builder = None
return new_builder
def buildbot_graph_builder(builders, revision, complete=True):
""" Return graph of builders based on a list of builders.
# XXX: It would be better if had a BuildbotGraph class instead of messing
with dictionaries.
https://github.com/mozilla/mozilla_ci_tools/issues/353
Input: a list of builders and a revision
Output: a set which includes a graph with the builders we received, the necessary upstream
jobs and list of builders which can been used to schedule jobs through trigger_job()
NOTE: We will make it only return builder graph once Buildbot jobs are scheduled via
the Buildbot Bridge (BBB) instead of the normal Buildbot scheduling.
Graph of N levels:
{
'Builder a1': {
'Builder a2': {
...
'Builder aN': None
},
},
'Builder b1': None
}
:param builders: List of builder names
:type builders: list
:param revision: push revision
:type revision: str
:return: A graph of buildernames (single level of graphs)
:rtype: dict
:param complete: indicate that build has been completed or not
:type: booleans
"""
graph = {}
ready_to_trigger = []
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for b in builders:
if not valid_builder(buildername=b, quiet=True):
continue
if is_downstream(b):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective = determine_trigger_objective(revision, b)[0]
# The build job is already completed, we can trigger the test job
if objective == b:
# XXX: Fix me - Adding test jobs to the graph without a build associated
# to it does not work. This will be fixed once we switch to scheduling
# Buildbot jobs through BBB
if complete:
graph[b] = None
else:
ready_to_trigger.append(b)
# The build job is running, there is nothing we can do
elif objective is None:
pass
# We need to trigger the build job and the test job
else:
if objective not in graph:
graph[objective] = {}
graph[objective][b] = None
else:
if b not in graph:
graph[b] = {}
# We might have left a build job poiting to an empty dict
for builder in graph:
if graph[builder] == {}:
graph[builder] = None
return graph, ready_to_trigger
def _create_task(buildername, repo_name, revision, metadata=None, task_graph_id=None,
parent_task_id=None, requires=None, properties={}, *args, **kwargs):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param metadata: Metadata for the task. If not specified, generate it.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError(
"The builder '%s' should be for repo: %s." % (buildername, repo_name)
)
repo_url = query_repo_url(repo_name)
push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
full_revision = str(push_info.changesets[0].node)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info.user,
}
all_properties.update(properties)
all_properties.update(get_builder_extra_properties(buildername))
metadata = metadata if metadata is not None else \
generate_metadata(repo_name=repo_name,
revision=revision,
name=buildername)
# The task's name is used in the task-graph-inspector to list all tasks
# and using the buildername makes it easy for a person to recognize each job.
metadata['name'] = buildername
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': full_revision
},
'properties': all_properties,
},
metadata=metadata,
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties']['parent_task_id'] = parent_task_id
return task
def _create_task(buildername,
repo_name,
revision,
task_graph_id=None,
parent_task_id=None,
requires=None,
properties={}):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError("The builder '%s' should be for repo: %s." %
(buildername, repo_name))
repo_url = query_repo_url(repo_name)
push_info = query_revision_info(repo_url, revision)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info['user'],
}
all_properties.update(properties)
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': revision
},
'properties': all_properties,
},
metadata_name=buildername)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties'][
'parent_task_id'] = parent_task_id
return task
def _create_task(buildername, repo_name, revision, task_graph_id=None,
parent_task_id=None, requires=None, properties={}):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError(
"The builder '%s' should be for repo: %s." % (buildername, repo_name)
)
repo_url = query_repo_url(repo_name)
push_info = query_revision_info(repo_url, revision)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info['user'],
}
all_properties.update(properties)
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': revision
},
'properties': all_properties,
},
metadata_name=buildername
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties']['parent_task_id'] = parent_task_id
return task
def test_valid_builder(self, fetch_allthethings_data):
fetch_allthethings_data.return_value = ALLTHETHINGS
buildername = "Windows 7 VM 32-bit mozilla-inbound pgo test mochitest-browser-chrome-1"
assert valid_builder(buildername)
def test_valid_builder(self, fetch_allthethings_data):
fetch_allthethings_data.return_value = ALLTHETHINGS
buildername = 'Windows 7 VM 32-bit mozilla-inbound pgo test mochitest-browser-chrome-1'
assert valid_builder(buildername)
def buildbot_graph_builder(builders, revision, complete=True):
""" Return graph of builders based on a list of builders.
# XXX: It would be better if had a BuildbotGraph class instead of messing
with dictionaries.
https://github.com/mozilla/mozilla_ci_tools/issues/353
Input: a list of builders and a revision
Output: a set which includes a graph with the builders we received, the necessary upstream
jobs and list of builders which can been used to schedule jobs through trigger_job()
NOTE: We will make it only return builder graph once Buildbot jobs are scheduled via
the Buildbot Bridge (BBB) instead of the normal Buildbot scheduling.
Graph of N levels:
{
'Builder a1': {
'Builder a2': {
...
'Builder aN': None
},
},
'Builder b1': None
}
:param builders: List of builder names
:type builders: list
:param revision: push revision
:type revision: str
:return: A graph of buildernames (single level of graphs)
:rtype: dict
:param complete: indicate that build has been completed or not
:type: booleans
"""
graph = {}
ready_to_trigger = []
# We need to determine what upstream jobs need to be triggered besides the
# builders already on our list
for b in builders:
if not valid_builder(buildername=b, quiet=True):
continue
if is_downstream(b):
# For test jobs, determine_trigger_objective()[0] can be 3 things:
# - the build job, if no build job exists
# - the test job, if the build job is already completed
# - None, if the build job is running
objective = determine_trigger_objective(revision, b)[0]
# The build job is already completed, we can trigger the test job
if objective == b:
# XXX: Fix me - Adding test jobs to the graph without a build associated
# to it does not work. This will be fixed once we switch to scheduling
# Buildbot jobs through BBB
if complete:
graph[b] = None
else:
ready_to_trigger.append(b)
# The build job is running, there is nothing we can do
elif objective is None:
pass
# We need to trigger the build job and the test job
else:
if objective not in graph:
graph[objective] = {}
graph[objective][b] = None
else:
if b not in graph:
graph[b] = {}
# We might have left a build job poiting to an empty dict
for builder in graph:
if graph[builder] == {}:
graph[builder] = None
return graph, ready_to_trigger
def _create_task(buildername,
repo_name,
revision,
metadata=None,
task_graph_id=None,
parent_task_id=None,
requires=None,
properties={},
*args,
**kwargs):
"""Return takcluster task to trigger a buildbot builder.
This function creates a generic task with the minimum amount of
information required for the buildbot-bridge to consider it valid.
You can establish a list dependencies to other tasks through the requires
field.
:param buildername: The name of a buildbot builder.
:type buildername: str
:param repo_name: The name of a repository e.g. mozilla-inbound
:type repo_name: str
:param revision: Changeset ID of a revision.
:type revision: str
:param metadata: Metadata for the task. If not specified, generate it.
:type metadata: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param parent_task_id: Task from which to find artifacts. It is not a dependency.
:type parent_task_id: str
:param requires: List of taskIds of other tasks which this task depends on.
:type requires: list
:returns: TaskCluster graph
:rtype: dict
"""
if not valid_builder(buildername):
raise MozciError("The builder '%s' is not a valid one." % buildername)
builder_info = get_buildername_metadata(buildername)
if builder_info['repo_name'] != repo_name:
raise MozciError("The builder '%s' should be for repo: %s." %
(buildername, repo_name))
repo_url = query_repo_url(repo_name)
push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
full_revision = str(push_info.changesets[0].node)
# Needed because of bug 1195751
all_properties = {
'product': builder_info['product'],
'who': push_info.user,
}
all_properties.update(properties)
all_properties.update(get_builder_extra_properties(buildername))
metadata = metadata if metadata is not None else \
generate_metadata(repo_name=repo_name,
revision=revision,
name=buildername)
# The task's name is used in the task-graph-inspector to list all tasks
# and using the buildername makes it easy for a person to recognize each job.
metadata['name'] = buildername
# XXX: We should validate that the parent task is a valid parent platform
# e.g. do not schedule Windows tests against Linux builds
task = create_task(
repo_name=repo_name,
revision=revision,
taskGroupId=task_graph_id,
workerType='buildbot-bridge',
provisionerId='buildbot-bridge',
payload={
'buildername': buildername,
'sourcestamp': {
'branch': repo_name,
'revision': full_revision
},
'properties': all_properties,
},
metadata=metadata,
)
if requires:
task['requires'] = requires
# Setting a parent_task_id as a property allows Mozharness to
# determine the artifacts we need for this job to run properly
if parent_task_id:
task['task']['payload']['properties'][
'parent_task_id'] = parent_task_id
return task
