def find_buildernames(repo, test=None, platform=None, job_type='opt'):
"""
Return a list of buildernames matching the criteria.
1) if the developer provides test, repo and platform and job_type
return only the specific buildername
2) if the developer provides test and platform only, then return
the test on all platforms
3) if the developer provides platform and repo, then return all
the tests on that platform
"""
assert test is not None or platform is not None, 'test and platform cannot both be None.'
buildernames = _filter_builders_matching(
fetch_allthethings_data()['builders'].keys(), ' %s ' % repo)
if test is not None:
buildernames = _filter_builders_matching(buildernames, test)
# Even when test is None we still only want test jobs
else:
buildernames = filter(lambda x: is_downstream(x), buildernames)
if platform is not None:
buildernames = filter(
lambda x: get_associated_platform_name(x) == platform,
buildernames)
if job_type is not None:
buildernames = filter(lambda x: _get_job_type(x) == job_type,
buildernames)
return buildernames
def find_buildernames(repo, test=None, platform=None, job_type='opt'):
"""
Return a list of buildernames matching the criteria.
1) if the developer provides test, repo and platform and job_type
return only the specific buildername
2) if the developer provides test and platform only, then return
the test on all platforms
3) if the developer provides platform and repo, then return all
the tests on that platform
"""
assert test is not None or platform is not None, 'test and platform cannot both be None.'
buildernames = _filter_builders_matching(fetch_allthethings_data()['builders'].keys(),
' %s ' % repo)
if test is not None:
buildernames = _filter_builders_matching(buildernames, test)
# Even when test is None we still only want test jobs
else:
buildernames = filter(lambda x: is_downstream(x), buildernames)
if platform is not None:
buildernames = filter(lambda x: get_associated_platform_name(x) == platform, buildernames)
if job_type is not None:
buildernames = filter(lambda x: _get_job_type(x) == job_type, buildernames)
return buildernames
def get_associated_platform_name(buildername):
"""Given a buildername, find the platform in which it is ran."""
props = fetch_allthethings_data()['builders'][buildername]['properties']
# For talos tests we have to check stage_platform
if 'talos' in buildername:
return props['stage_platform']
else:
return props['platform']
def get_associated_platform_name(buildername):
"""Given a buildername, find the platform in which it is ran."""
props = fetch_allthethings_data()['builders'][buildername]['properties']
# For talos tests we have to check stage_platform
if 'talos' in buildername:
return props['stage_platform']
else:
return props['platform']
def is_downstream(buildername):
"""Determine if a job requires files to be triggered."""
# Builders in gaia-try are at same time build and test jobs, and
# should be considered upstream.
if " gaia-try " in buildername:
return False
props = fetch_allthethings_data()['builders'][buildername]['properties']
return 'slavebuilddir' in props and props['slavebuilddir'] == 'test'
def is_downstream(buildername):
"""Determine if a job requires files to be triggered."""
# Builders in gaia-try are at same time build and test jobs, and
# should be considered upstream.
if " gaia-try " in buildername:
return False
props = fetch_allthethings_data()['builders'][buildername]['properties']
return 'slavebuilddir' in props and props['slavebuilddir'] == 'test'
def _process_data():
"""Filling the dictionaries used by determine_upstream_builder."""
# We check if we already computed before
if BUILDERNAME_TO_TRIGGER:
LOG.debug(
"Reusing builders' relations computed from allthethings data.")
return
LOG.debug("Computing builders' relations from allthethings data.")
# We'll look at every builder and if it's a build job we will add it
# to SHORTNAME_TO_NAME
for buildername, builderinfo in fetch_allthethings_data(
)['builders'].iteritems():
if not is_downstream(buildername):
SHORTNAME_TO_NAME[builderinfo['shortname']] = buildername
BUILD_JOBS[buildername.lower()] = buildername
# data['schedulers'] is a dictionary that maps a scheduler name to a
# dictionary of it's properties:
# "schedulers": {...
# "tests-larch-panda_android-opt-unittest": {
# "downstream": [ "Android 4.0 armv7 API 11+ larch opt test cppunit",
# "Android 4.0 armv7 API 11+ larch opt test crashtest",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-1",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-2",
# ... ],
# "triggered_by": ["larch-android-api-11-opt-unittest"]},
# A test scheduler has a list of tests in "downstream" and a trigger
# name in "triggered_by". We will map every test in downstream to the
# trigger name in triggered_by
for sched, values in fetch_allthethings_data()['schedulers'].iteritems():
# We are only interested in test schedulers
if not sched.startswith('tests-'):
continue
for buildername in values['downstream']:
assert buildername.lower() not in BUILDERNAME_TO_TRIGGER
BUILDERNAME_TO_TRIGGER[
buildername.lower()] = values['triggered_by'][0]
def _process_data():
"""Filling the dictionaries used by determine_upstream_builder."""
# We check if we already computed before
if BUILDERNAME_TO_TRIGGER:
LOG.debug("Reusing builders' relations computed from allthethings data.")
return
LOG.debug("Computing builders' relations from allthethings data.")
# We'll look at every builder and if it's a build job we will add it
# to SHORTNAME_TO_NAME
for buildername, builderinfo in fetch_allthethings_data()['builders'].iteritems():
if not is_downstream(buildername):
SHORTNAME_TO_NAME[builderinfo['shortname']] = buildername
BUILD_JOBS[buildername.lower()] = buildername
# data['schedulers'] is a dictionary that maps a scheduler name to a
# dictionary of it's properties:
# "schedulers": {...
# "tests-larch-panda_android-opt-unittest": {
# "downstream": [ "Android 4.0 armv7 API 11+ larch opt test cppunit",
# "Android 4.0 armv7 API 11+ larch opt test crashtest",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-1",
# "Android 4.0 armv7 API 11+ larch opt test jsreftest-2",
# ... ],
# "triggered_by": ["larch-android-api-11-opt-unittest"]},
# A test scheduler has a list of tests in "downstream" and a trigger
# name in "triggered_by". We will map every test in downstream to the
# trigger name in triggered_by
for sched, values in fetch_allthethings_data()['schedulers'].iteritems():
# We are only interested in test schedulers
if not sched.startswith('tests-'):
continue
for buildername in values['downstream']:
assert buildername.lower() not in BUILDERNAME_TO_TRIGGER
BUILDERNAME_TO_TRIGGER[buildername.lower()] = values['triggered_by'][0]
def build_talos_buildernames_for_repo(repo_name, pgo_only=False):
"""
This function aims to generate all possible talos jobs for a given branch.
Here we take the list of talos buildernames for a given branch. When
we want pgo, we build a list of pgo buildernames, then find the non-pgo builders
which do not have a pgo equivalent. To do this, we hack the buildernames in
a temporary set by removing ' pgo' from the name, then finding the unique jobs
in the talos_re jobs. Now we can take the pgo jobs and jobs with no pgo
equivalent and have a full set of pgo jobs.
"""
buildernames = fetch_allthethings_data()['builders']
retVal = []
# Android and OSX do not have PGO, so we need to get those specific jobs
pgo_re = re.compile(".*%s pgo talos.*" % repo_name)
talos_re = re.compile(".*%s talos.*" % repo_name)
talos_jobs = set()
pgo_jobs = set()
tp_jobs = set()
for builder in buildernames:
if talos_re.match(builder):
talos_jobs.add(builder)
if pgo_re.match(builder):
pgo_jobs.add(builder)
tp_jobs.add(builder.replace(' pgo', ''))
if pgo_only:
non_pgo_jobs = talos_jobs - tp_jobs
talos_jobs = pgo_jobs.union(non_pgo_jobs)
for builder in talos_jobs:
# This is a temporary hack to not trigger 'Windows 10' jobs on try.
# Remove it when not necessary.
if 'Windows 10' in builder:
continue
retVal.append(builder)
retVal.sort()
return retVal
def build_talos_buildernames_for_repo(repo_name, pgo_only=False):
"""
This function aims to generate all possible talos jobs for a given branch.
Here we take the list of talos buildernames for a given branch. When
we want pgo, we build a list of pgo buildernames, then find the non-pgo builders
which do not have a pgo equivalent. To do this, we hack the buildernames in
a temporary set by removing ' pgo' from the name, then finding the unique jobs
in the talos_re jobs. Now we can take the pgo jobs and jobs with no pgo
equivalent and have a full set of pgo jobs.
"""
buildernames = fetch_allthethings_data()['builders']
retVal = []
# Android and OSX do not have PGO, so we need to get those specific jobs
pgo_re = re.compile(".*%s pgo talos.*" % repo_name)
talos_re = re.compile(".*%s talos.*" % repo_name)
talos_jobs = set()
pgo_jobs = set()
tp_jobs = set()
for builder in buildernames:
if talos_re.match(builder):
talos_jobs.add(builder)
if pgo_re.match(builder):
pgo_jobs.add(builder)
tp_jobs.add(builder.replace(' pgo', ''))
if pgo_only:
non_pgo_jobs = talos_jobs - tp_jobs
talos_jobs = pgo_jobs.union(non_pgo_jobs)
for builder in talos_jobs:
# This is a temporary hack to not trigger 'Windows 10' jobs on try.
# Remove it when not necessary.
if 'Windows 10' in builder:
continue
retVal.append(builder)
retVal.sort()
return retVal
def get_builder_information(buildername):
"""Return all metadata from allthethings associated to a builder."""
return fetch_allthethings_data()['builders'][buildername]
def get_builder_information(buildername):
"""Return all metadata from allthethings associated to a builder."""
return fetch_allthethings_data()['builders'][buildername]
