def test_calling_twice_without_caching(self, head, get):
"""Without caching, get and head should both be called 2 times."""
self.assertEquals(allthethings.fetch_allthethings_data(no_caching=True), self.expected)
head.assert_called_with(self.URL)
get.assert_called_with(self.URL, stream=True)
# Calling again
self.assertEquals(allthethings.fetch_allthethings_data(no_caching=True), self.expected)
assert get.call_count == 2
assert head.call_count == 2
def _wanted_builder(builder, filter=True, repo_name=None):
""" Filter unnecessary builders that Buildbot's setup has. """
if filter:
# Builders to ignore
if builder.startswith('release-') or \
builder.endswith('bundle'):
return False
# We lack metadata in allthethings for release builders
# in order to call get_buildername_metadata()
info = get_buildername_metadata(builder)
if repo_name and repo_name != info['repo_name']:
return False
# Exclude the non pgo talos builders for m-a and m-b
# *if* there is no pgo talos builder
# This is to work around bug 1149514
if filter:
if info['repo_name'] in ('mozilla-aurora', 'mozilla-beta') and \
info['job_type'] == 'talos' and _get_job_type(builder) == 'opt':
equiv_pgo_builder = builder.replace('talos', 'pgo talos')
if equiv_pgo_builder in fetch_allthethings_data()['builders']:
# There are two talos builders, we only can use the pgo one
return False
else:
# This platform only has only one talos builder
return True
return True
def test_with_verify_set_to_false_and_existing_cache(self):
"""If verify is set to False and there already is a file, we should just use it."""
# Making sure the file exists.
with open(TMP_FILENAME, 'w') as f:
f.write('{"data": 2}')
self.assertEquals(allthethings.fetch_allthethings_data(verify=False),
{'data': 2})
def _wanted_builder(builder, filter=True):
""" Filter unnecessary builders that Buildbot's setup has.
If you call the function without filter is the same as always returning True.
"""
if not filter:
# XXX: revisit why we allow for this option
return True
else:
# We lack metadata in allthethings for release builders
# in order to call get_buildername_metadata()
# We don't care about the hg bundle builders
if builder.startswith('release-') or \
builder.endswith('bundle'):
return False
info = get_buildername_metadata(builder)
# On aurora and beta we have pgo and opt builders for talos even though
# there is no build to trigger the pgo builders. Exclude those builders as valid
# This is to work around bug 1149514
if info['repo_name'] in ('mozilla-aurora', 'mozilla-beta') and \
info['job_type'] == 'talos':
equiv_pgo_builder = builder.replace('talos', 'pgo talos')
if equiv_pgo_builder in fetch_allthethings_data()['builders']:
# There are two talos builders, we only can use the pgo one
return False
return True
def _wanted_builder(builder, filter=True, repo_name=None):
""" Filter unnecessary builders that Buildbot's setup has. """
if filter:
# Builders to ignore
if builder.startswith('release-') or \
builder.endswith('bundle'):
return False
# We lack metadata in allthethings for release builders
# in order to call get_buildername_metadata()
info = get_buildername_metadata(builder)
if repo_name and repo_name != info['repo_name']:
return False
# Exclude the non pgo talos builders for m-a and m-b
# *if* there is no pgo talos builder
# This is to work around bug 1149514
if filter:
if info['repo_name'] in ('mozilla-aurora', 'mozilla-beta') and \
info['job_type'] == 'talos' and _get_job_type(builder) == 'opt':
equiv_pgo_builder = builder.replace('talos', 'pgo talos')
if equiv_pgo_builder in fetch_allthethings_data()['builders']:
# There are two talos builders, we only can use the pgo one
return False
else:
# This platform only has only one talos builder
return True
return True
def _wanted_builder(builder, filter=True):
""" Filter unnecessary builders that Buildbot's setup has.
If you call the function without filter is the same as always returning True.
"""
if not filter:
# XXX: revisit why we allow for this option
return True
else:
# We lack metadata in allthethings for release builders
# in order to call get_buildername_metadata()
# We don't care about the hg bundle builders
if builder.startswith('release-') or \
builder.endswith('bundle'):
return False
info = get_buildername_metadata(builder)
# On aurora and beta we have pgo and opt builders for talos even though
# there is no build to trigger the pgo builders. Exclude those builders as valid
# This is to work around bug 1149514
if info['repo_name'] in ('mozilla-aurora', 'mozilla-beta') and \
info['job_type'] == 'talos':
equiv_pgo_builder = builder.replace('talos', 'pgo talos')
if equiv_pgo_builder in fetch_allthethings_data()['builders']:
# There are two talos builders, we only can use the pgo one
return False
return True
def _wanted_builder(builder, filter=True, repo_name=None):
""" Filter unnecessary builders that Buildbot's setup has. """
if filter:
# Builders to ignore
if builder.startswith("release-") or builder.endswith("bundle"):
return False
# We lack metadata in allthethings for release builders
# in order to call get_buildername_metadata()
info = get_buildername_metadata(builder)
if repo_name and repo_name != info["repo_name"]:
return False
# Exclude the non pgo talos builders for m-a and m-b
# *if* there is no pgo talos builder
# This is to work around bug 1149514
if filter:
if (
info["repo_name"] in ("mozilla-aurora", "mozilla-beta")
and info["job_type"] == "talos"
and _get_job_type(builder) == "opt"
):
equiv_pgo_builder = builder.replace("talos", "pgo talos")
if equiv_pgo_builder in fetch_allthethings_data()["builders"]:
# There are two talos builders, we only can use the pgo one
return False
else:
# This platform only has only one talos builder
return True
if (
info["repo_name"]
not in ("mozilla-aurora", "mozilla-beta", "mozilla-esr38", "mozilla-esr45", "mozilla-release")
and info["job_type"] == "test"
and _get_job_type(builder) == "pgo"
):
# This displays opt builder amongst opt and pgo for mozilla-inbound,central etc.
# and shows both pgo and opt for mozilla-aurora,beta,esr{38,45},release.
equiv_opt_builder = builder.replace("pgo", "opt")
if equiv_opt_builder in fetch_allthethings_data()["builders"]:
return False
else:
return True
return True
def test_calling_twice_with_caching(self, head, get):
"""
We are going to call fetch_allthethings_data 2 times.
The first time it should use requests.get to download the file, and requests.head
to verify its integrity. The second time it will return the variable stored in-memory,
so it won't call neither head or get.
"""
# Calling the function the first time, and checking its result
self.assertEquals(allthethings.fetch_allthethings_data(), self.expected)
# Calling again
allthethings.fetch_allthethings_data()
get.assert_called_with(self.URL, stream=True)
assert get.call_count == 1
head.assert_called_with(self.URL)
assert head.call_count == 1
def test_calling_with_bad_cache(self, head, get):
"""If the existing file is bad, we should download a new one."""
# Making sure the cache exists and it's bad
with open(TMP_FILENAME, 'w') as f:
f.write('bad file')
self.assertEquals(allthethings.fetch_allthethings_data(), self.expected)
head.assert_called_with(self.URL)
get.assert_called_with(self.URL, stream=True)
def _process_data():
"""Filling the dictionaries used by determine_upstream_builder."""
# We check if we already computed before
if BUILDERNAME_TO_TRIGGER:
return
LOG.debug("Computing builders' relations from allthethings data.")
# We'll look at every builder and if it's a build job we will add it
# to SHORTNAME_TO_NAME
for buildername, builderinfo in fetch_allthethings_data(
)['builders'].iteritems():
if not _wanted_builder(buildername):
continue
if is_upstream(buildername):
SHORTNAME_TO_NAME[builderinfo['shortname']] = buildername
BUILD_JOBS[buildername.lower()] = buildername
# data['schedulers'] is a dictionary that maps a scheduler name to a
# dictionary of it's properties:
# "schedulers": {...
# "tests-larch-panda_android-opt-unittest": {
# "downstream": [ "Android 4.0 armv7 API 11+ larch opt test cppunit",
# "Android 4.0 armv7 API 11+ larch opt test crashtest",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-1",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-2",
# ... ],
# "triggered_by": ["larch-android-api-11-opt-unittest"]},
# A test scheduler has a list of tests in "downstream" and a trigger
# name in "triggered_by". We will map every test in downstream to the
# trigger name in triggered_by
for sched, values in fetch_allthethings_data()['schedulers'].iteritems():
# We are only interested in test schedulers
if not sched.startswith('tests-'):
continue
for buildername in values['downstream']:
assert buildername.lower() not in BUILDERNAME_TO_TRIGGER
BUILDERNAME_TO_TRIGGER[
buildername.lower()] = values['triggered_by'][0]
def list_builders(repo_name=None, filter=True):
"""Return a list of all builders running in the buildbot CI."""
all_builders = fetch_allthethings_data()['builders']
assert len(all_builders) > 0, "The list of builders cannot be empty."
# Let's filter out builders which are not triggered per push
# and are not associated to a repo_name if set
builders_list = []
for builder in all_builders.keys():
if _wanted_builder(builder=builder, filter=filter, repo_name=repo_name):
builders_list.append(builder)
return builders_list
def _process_data():
"""Filling the dictionaries used by determine_upstream_builder."""
# We check if we already computed before
if BUILDERNAME_TO_TRIGGER:
return
LOG.debug("Computing builders' relations from allthethings data.")
# We'll look at every builder and if it's a build job we will add it
# to SHORTNAME_TO_NAME
for buildername, builderinfo in fetch_allthethings_data()['builders'].iteritems():
if not _wanted_builder(buildername):
continue
if is_upstream(buildername):
SHORTNAME_TO_NAME[builderinfo['shortname']] = buildername
BUILD_JOBS[buildername.lower()] = buildername
# data['schedulers'] is a dictionary that maps a scheduler name to a
# dictionary of it's properties:
# "schedulers": {...
# "tests-larch-panda_android-opt-unittest": {
# "downstream": [ "Android 4.0 armv7 API 11+ larch opt test cppunit",
# "Android 4.0 armv7 API 11+ larch opt test crashtest",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-1",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-2",
# ... ],
# "triggered_by": ["larch-android-api-11-opt-unittest"]},
# A test scheduler has a list of tests in "downstream" and a trigger
# name in "triggered_by". We will map every test in downstream to the
# trigger name in triggered_by
for sched, values in fetch_allthethings_data()['schedulers'].iteritems():
# We are only interested in test schedulers
if not sched.startswith('tests-'):
continue
for buildername in values['downstream']:
assert buildername.lower() not in BUILDERNAME_TO_TRIGGER
BUILDERNAME_TO_TRIGGER[buildername.lower()] = values['triggered_by'][0]
def list_builders(repo_name=None, filter=True):
"""Return a list of all builders running in the buildbot CI."""
all_builders = fetch_allthethings_data()['builders']
assert len(all_builders) > 0, "The list of builders cannot be empty."
# Let's filter out builders which are not triggered per push
# and are not associated to a repo_name if set
builders_list = []
for builder in all_builders.keys():
if _wanted_builder(builder=builder, filter=filter,
repo_name=repo_name):
builders_list.append(builder)
return builders_list
def test_no_testers_without_builders():
# We need to assert that allthethings.json has been generated
# before running this test rather than fetching the one from the server
# This is important as we want to test against the latest list of builders
assert os.path.exists("allthethings.json")
orphan_builders = []
j = fetch_allthethings_data(verify=False)
builders = j["builders"].keys()
assert builders is not None, "The list of builders cannot be empty."
for builder in sorted(builders):
if determine_upstream_builder(builder) is None:
orphan_builders.append(builder)
assert len(orphan_builders) == 0, \
"There are downstream builders without upstream builders to trigger them."
def get_SETA_interval_dict(force=False):
"""
Return dictionary with SETA intervals for buildernames.
SETA intervals are in the form of [7, 3600] where 7 represents the number of pushes after
SETA runs and 3600 represents time interval in seconds after which SETA runs
if number of pushes isn't reached.
:param force: Default False. Forces refresh of SETA dict.
:type force: boolean
:returns: Returns dict with buildernames as keys and SETA interval as value.
:rtype: dict
"""
global SETA_DICT
if SETA_DICT and not force:
return SETA_DICT
SETA_DICT = {}
for sched, values in fetch_allthethings_data()['schedulers'].iteritems():
# We are only interested in test schedulers
# Example scheduler names:
# tests-fx-team-snowleopard-debug-unittest-7-3600
# tests-fx-team-snowleopard-opt-unittest-7-3600
if not sched.startswith('tests-'):
continue
sched_str_list = sched.split('-')
# Only schedulers with SETA information have a numeric value
# [u'tests', u'fx', u'team', u'snowleopard', u'opt', u'unittest', u'7', u'3600']
# [u'tests', u'fx', u'team', u'ubuntu64_vm', u'opt', u'unittest']
# We can call isnumeric because it is a unicode value
if sched_str_list[-1].isnumeric():
pushes = int(sched_str_list[-2])
seconds = int(sched_str_list[-1])
# Iterate over all the downstream builders this scheduler schedules
for buildername in values['downstream']:
SETA_DICT[buildername] = [pushes, seconds]
return SETA_DICT
def get_SETA_interval_dict(force=False):
"""
Return dictionary with SETA intervals for buildernames.
SETA intervals are in the form of [7, 3600] where 7 represents the number of pushes after
SETA runs and 3600 represents time interval in seconds after which SETA runs
if number of pushes isn't reached.
:param force: Default False. Forces refresh of SETA dict.
:type force: boolean
:returns: Returns dict with buildernames as keys and SETA interval as value.
:rtype: dict
"""
global SETA_DICT
if SETA_DICT and not force:
return SETA_DICT
SETA_DICT = {}
for sched, values in fetch_allthethings_data()['schedulers'].iteritems():
# We are only interested in test schedulers
# Example scheduler names:
# tests-fx-team-snowleopard-debug-unittest-7-3600
# tests-fx-team-snowleopard-opt-unittest-7-3600
if not sched.startswith('tests-'):
continue
sched_str_list = sched.split('-')
# Only schedulers with SETA information have a numeric value
# [u'tests', u'fx', u'team', u'snowleopard', u'opt', u'unittest', u'7', u'3600']
# [u'tests', u'fx', u'team', u'ubuntu64_vm', u'opt', u'unittest']
# We can call isnumeric because it is a unicode value
if sched_str_list[-1].isnumeric():
pushes = int(sched_str_list[-2])
seconds = int(sched_str_list[-1])
# Iterate over all the downstream builders this scheduler schedules
for buildername in values['downstream']:
SETA_DICT[buildername] = [pushes, seconds]
return SETA_DICT
def get_SETA_interval_dict(force=False):
"""
Return dictionary with SETA intervals for buildernames.
SETA intervals is in form [7, 3600] where 7 represents number of pushes after
SETA runs and 3600 represents time interval in seconds after which SETA runs
if number of pushes isn't reached.
:param force: Default False. Forces refresh of SETA dict.
:type force: boolean
:returns: Returns dict with buildernames as keys and SETA interval as value.
:rtype: dict
"""
global SETA_DICT
if SETA_DICT and not force:
return SETA_DICT
SETA_DICT = {}
for sched, values in fetch_allthethings_data()["schedulers"].iteritems():
# We are only interested in test schedulers
# Example scheduler names:
# tests-fx-team-snowleopard-debug-unittest-7-3600
# tests-fx-team-snowleopard-opt-unittest-7-3600
if not sched.startswith("tests-"):
continue
sched_str_list = sched.split("-")
try:
for buildername in values["downstream"]:
seta = []
seta.append(int(sched_str_list[-2]))
seta.append(int(sched_str_list[-1]))
SETA_DICT[buildername] = seta
except (ValueError, IndexError):
continue
return SETA_DICT
def test_no_testers_without_builders():
# We need to assert that allthethings.json has been generated
# before running this test rather than fetching the one from the server
# This is important as we want to test against the latest list of builders
assert os.path.exists("allthethings.json")
orphan_builders = []
j = fetch_allthethings_data(verify=False)
builders = j["builders"].keys()
assert builders is not None, "The list of builders cannot be empty."
for builder in sorted(builders):
# Bug 1330680 - patches to disable bb nightlies on linux32/linux64/android
# there are talos jobs that don't have associated build jobs as part
# of tc migration
if 'talos' in builder and 'Ubuntu HW' in builder:
return
if determine_upstream_builder(builder) is None:
orphan_builders.append(builder)
assert len(orphan_builders) == 0, \
"There are downstream builders without upstream builders to trigger them."
def test_with_verify_set_to_false_and_no_cache(self):
"""If verify is set to False and there is no file, it should raise an Error."""
with self.assertRaises(AssertionError):
allthethings.fetch_allthethings_data(verify=False)
"""This script writes a mapping from platforms to tests that run in it to graph.json."""
import ujson as json
from mozci.platforms import build_tests_per_platform_graph, _filter_builders_matching
from mozci.sources.allthethings import fetch_allthethings_data
from mozci.utils.transfer import path_to_file
if __name__ == '__main__':
with open(path_to_file('graph.json'), 'w') as f:
builders = _filter_builders_matching(fetch_allthethings_data()['builders'], " try ")
graph = build_tests_per_platform_graph(builders)
json.dump(graph, f, sort_keys=True, indent=4, separators=(',', ': '))
"""This script writes a mapping from platforms to tests that run in it to graph.json."""
import json
from mozci.platforms import build_tests_per_platform_graph, _filter_builders_matching
from mozci.sources.allthethings import fetch_allthethings_data
from mozci.utils.transfer import path_to_file
if __name__ == '__main__':
with open(path_to_file('graph.json'), 'w') as f:
builders = _filter_builders_matching(
fetch_allthethings_data()['builders'], " try ")
graph = build_tests_per_platform_graph(builders)
json.dump(graph, f, sort_keys=True, indent=4, separators=(',', ': '))
def get_buildername_metadata(buildername):
"""Return metadata associated to a buildername.
Returns a dictionary with the following information:
* build_type - It returns 'opt' or 'debug'
* downstream - If the job requires an upstream job to be triggered
* job_type - It returns 'build', 'test' or 'talos'
* platform_name - It associates upstream & downstream builders (e.g. win32)
* product - e.g. firefox
* repo_name - Associated short name for a repository (e.g. alder)
* suite_name - talos & test jobs have an associated suite name (e.g chromez)
"""
if buildername not in fetch_allthethings_data()["builders"]:
raise MozciError
props = _get_raw_builder_metadata(buildername)["properties"]
# For talos tests we have to check stage_platform
if "talos" in buildername:
platform_name = props["stage_platform"]
job_type = "talos"
# 'Windows 7 32-bit mozilla-central pgo talos chromez-e10s' -> chromez-e10s
suite_name = buildername.split(" ")[-1]
elif "test" in buildername:
job_type = "test"
platform_name = props["platform"]
suite_name = buildername.split(" ")[-1]
else:
job_type = "build"
platform_name = props["platform"]
suite_name = None
if platform_name.endswith("-debug"):
# e.g. win32-debug
platform_name = platform_name[: -len("-debug")]
build_type = "debug"
else:
build_type = "opt"
repo_path = props.get("branch")
repo_name = _get_repo_name(repo_path)
# Builders in gaia-try are at same time build and test jobs, and
# should be considered upstream.
if repo_name == "gaia-try":
downstream = False
else:
downstream = "slavebuilddir" in props and props["slavebuilddir"] == "test"
assert build_type is not None
assert repo_name is not None
assert repo_path is not None
assert platform_name is not None
return {
"build_type": build_type,
"downstream": downstream,
"job_type": job_type,
"platform_name": platform_name,
"product": props["product"],
"repo_name": repo_name,
"suite_name": suite_name,
}
def get_buildername_metadata(buildername):
"""Return metadata associated to a buildername.
Returns a dictionary with the following information:
* build_type - It returns 'opt' or 'debug'
* downstream - If the job requires an upstream job to be triggered
* job_type - It returns 'build', 'test' or 'talos'
* platform_name - It associates upstream & downstream builders (e.g. win32)
* product - e.g. firefox
* repo_name - Associated short name for a repository (e.g. alder)
* suite_name - talos & test jobs have an associated suite name (e.g chromez)
"""
if buildername not in fetch_allthethings_data()['builders']:
return None
props = _get_raw_builder_metadata(buildername)['properties']
# For talos tests we have to check stage_platform
if 'talos' in buildername:
platform_name = props['stage_platform']
job_type = 'talos'
# 'Windows 7 32-bit mozilla-central pgo talos chromez-e10s' -> chromez-e10s
suite_name = buildername.split(" ")[-1]
elif 'test' in buildername:
job_type = 'test'
platform_name = props['platform']
suite_name = buildername.split(" ")[-1]
else:
job_type = 'build'
platform_name = props['platform']
suite_name = None
if platform_name.endswith('-debug'):
# e.g. win32-debug
platform_name = platform_name[:-len('-debug')]
build_type = 'debug'
else:
build_type = 'opt'
repo_path = props.get('branch')
repo_name = _get_repo_name(repo_path)
# Builders in gaia-try are at same time build and test jobs, and
# should be considered upstream.
if repo_name == 'gaia-try':
downstream = False
else:
downstream = 'slavebuilddir' in props and props['slavebuilddir'] == 'test'
assert build_type is not None
assert repo_name is not None
assert repo_path is not None
assert platform_name is not None
return {
'build_type': build_type,
'downstream': downstream,
'job_type': job_type,
'platform_name': platform_name,
'product': props['product'],
'repo_name': repo_name,
'suite_name': suite_name,
}
def _get_raw_builder_metadata(buildername):
"""Return all metadata from allthethings associated to a builder."""
return fetch_allthethings_data()['builders'][buildername]
def get_buildername_metadata(buildername):
"""Return metadata associated to a buildername.
Returns a dictionary with the following information:
* build_type - It returns 'opt' or 'debug' or 'pgo'
* downstream - If the job requires an upstream job to be triggered
* job_type - It returns 'build', 'test' or 'talos'
* platform_name - It associates upstream & downstream builders (e.g. win32)
* product - e.g. firefox
* repo_name - Associated short name for a repository (e.g. alder)
* suite_name - talos & test jobs have an associated suite name (e.g chromez)
* nightly - Tells us whether a job is a nightly job
"""
if buildername not in fetch_allthethings_data()['builders']:
raise MissingBuilderError("Builder '{}' is missing. All builders' lenght: {}".format(
buildername, len(fetch_allthethings_data()['builders']))
)
props = _get_raw_builder_metadata(buildername)['properties']
meta = {
'downstream': 'slavebuilddir' in props and props['slavebuilddir'] == 'test',
'platform_name': props['platform'],
'product': props['product'],
'repo_name': _get_repo_name(props.get('branch')),
}
# Build jobs
if not meta['downstream']:
meta['job_type'] = 'build'
suite_name = None
else:
# e.g. 'Windows 7 32-bit mozilla-central pgo talos chromez-e10s' -> chromez-e10s
suite_name = str(buildername.split(" ")[-1])
# Talos jobs
if 'talos' in buildername:
meta['job_type'] = 'talos'
# For talos tests we have to check stage_platform instead of platform
meta['platform_name'] = props['stage_platform']
# Test jobs
elif 'test' in buildername:
meta['job_type'] = 'test'
ending = meta['platform_name'].split('-')[-1]
if ending in ('debug', 'opt', 'pgo'):
# e.g. win32-st-an-debug, emulator-debug, win32-debug
# build_type == 'debug'
# platform_name == ('win32-st-an', 'emulator', 'win32')
meta['build_type'] = ending
meta['platform_name'] = meta['platform_name'][:-len(ending) - 1]
elif 'debug' in buildername:
meta['build_type'] = 'debug'
elif 'opt' in buildername:
meta['build_type'] = 'opt'
elif 'pgo' in buildername:
meta['build_type'] = 'pgo'
elif meta['job_type'] == 'build':
# Release repositories *only* have PGO builds even though their name does not contain
# 'pgo' in the buildername
if meta['repo_name'] in ('mozilla-aurora', 'mozilla-beta', 'mozilla-release', 'esr'):
meta['build_type'] = 'pgo'
else:
meta['build_type'] = 'opt'
else:
# e.g. Rev7 MacOSX Yosemite 10.10.5 mozilla-beta talos other-e10s
meta['build_type'] = 'opt'
assert all(meta)
# Since builds don't have a suite name
meta['suite_name'] = suite_name
if 'nightly' in buildername:
meta['nightly'] = True
else:
meta['nightly'] = False
return meta
def _get_raw_builder_metadata(buildername):
"""Return all metadata from allthethings associated to a builder."""
return fetch_allthethings_data()['builders'][buildername]
def get_buildername_metadata(buildername):
"""Return metadata associated to a buildername.
Returns a dictionary with the following information:
* build_type - It returns 'opt' or 'debug' or 'pgo'
* downstream - If the job requires an upstream job to be triggered
* job_type - It returns 'build', 'test' or 'talos'
* platform_name - It associates upstream & downstream builders (e.g. win32)
* product - e.g. firefox
* repo_name - Associated short name for a repository (e.g. alder)
* suite_name - talos & test jobs have an associated suite name (e.g chromez)
* nightly - Tells us whether a job is a nightly job
"""
if buildername not in fetch_allthethings_data()['builders']:
raise MissingBuilderError(
"Builder '{}' is missing. All builders' length: {}".format(
buildername, len(fetch_allthethings_data()['builders'])))
props = _get_raw_builder_metadata(buildername)['properties']
meta = {
'downstream': 'slavebuilddir' in props
and props['slavebuilddir'] == 'test',
'platform_name': props['platform'],
'product': props['product'],
'repo_name': _get_repo_name(props.get('branch')),
}
# Build jobs
if not meta['downstream']:
meta['job_type'] = 'build'
suite_name = None
else:
# e.g. 'Windows 7 32-bit mozilla-central pgo talos chromez-e10s' -> chromez-e10s
suite_name = str(buildername.split(" ")[-1])
# Talos jobs
if 'talos' in buildername:
meta['job_type'] = 'talos'
# For talos tests we have to check stage_platform instead of platform
meta['platform_name'] = props['stage_platform']
# Test jobs
elif 'test' in buildername:
meta['job_type'] = 'test'
ending = meta['platform_name'].split('-')[-1]
if ending in ('debug', 'opt', 'pgo'):
# e.g. win32-st-an-debug, emulator-debug, win32-debug
# build_type == 'debug'
# platform_name == ('win32-st-an', 'emulator', 'win32')
meta['build_type'] = ending
meta['platform_name'] = meta['platform_name'][:-len(ending) - 1]
elif 'debug' in buildername:
meta['build_type'] = 'debug'
elif 'opt' in buildername:
meta['build_type'] = 'opt'
elif 'pgo' in buildername:
meta['build_type'] = 'pgo'
elif meta['job_type'] == 'build':
# Release repositories *only* have PGO builds even though their name does not contain
# 'pgo' in the buildername
if meta['repo_name'] in ('mozilla-aurora', 'mozilla-beta',
'mozilla-release', 'esr'):
meta['build_type'] = 'pgo'
else:
meta['build_type'] = 'opt'
else:
# e.g. Rev7 MacOSX Yosemite 10.10.5 mozilla-beta talos other-e10s
meta['build_type'] = 'opt'
assert all(meta)
# Since builds don't have a suite name
meta['suite_name'] = suite_name
if 'nightly' in buildername:
meta['nightly'] = True
else:
meta['nightly'] = False
return meta
def test_with_verify_set_to_false_and_existing_cache(self):
"""If verify is set to False and there already is a file, we should just use it."""
# Making sure the file exists.
with open(TMP_FILENAME, 'w') as f:
f.write('{"data": 2}')
self.assertEquals(allthethings.fetch_allthethings_data(verify=False), {'data': 2})
def get_buildername_metadata(buildername):
"""Return metadata associated to a buildername.
Returns a dictionary with the following information:
* build_type - It returns 'opt' or 'debug'
* downstream - If the job requires an upstream job to be triggered
* job_type - It returns 'build', 'test' or 'talos'
* platform_name - It associates upstream & downstream builders (e.g. win32)
* product - e.g. firefox
* repo_name - Associated short name for a repository (e.g. alder)
* suite_name - talos & test jobs have an associated suite name (e.g chromez)
"""
if buildername not in fetch_allthethings_data()['builders']:
return None
props = _get_raw_builder_metadata(buildername)['properties']
# For talos tests we have to check stage_platform
if 'talos' in buildername:
platform_name = props['stage_platform']
job_type = 'talos'
# 'Windows 7 32-bit mozilla-central pgo talos chromez-e10s' -> chromez-e10s
suite_name = buildername.split(" ")[-1]
elif 'test' in buildername:
job_type = 'test'
platform_name = props['platform']
suite_name = buildername.split(" ")[-1]
else:
job_type = 'build'
platform_name = props['platform']
suite_name = None
if platform_name.endswith('-debug'):
# e.g. win32-debug
platform_name = platform_name[:-len('-debug')]
build_type = 'debug'
else:
build_type = 'opt'
repo_path = props.get('branch')
repo_name = _get_repo_name(repo_path)
# Builders in gaia-try are at same time build and test jobs, and
# should be considered upstream.
if repo_name == 'gaia-try':
downstream = False
else:
downstream = 'slavebuilddir' in props and props[
'slavebuilddir'] == 'test'
assert build_type is not None
assert repo_name is not None
assert repo_path is not None
assert platform_name is not None
return {
'build_type': build_type,
'downstream': downstream,
'job_type': job_type,
'platform_name': platform_name,
'product': props['product'],
'repo_name': repo_name,
'suite_name': suite_name,
}