def _process(self, project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
if not job_priorities:
raise SetaError("Call docker-compose run backend ./manage.py initialize_seta")
jobs = []
ref_data_names_map = get_reference_data_names(project, build_system)
# now check the JobPriority table against the list of valid runnable
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning(
'Job priority key %s for (%s) not found in accepted jobs list', key, jp
)
return jobs
def _process(self, project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
jobs = []
# we cache the reference data names in order to reduce API calls
cache_key = '{}-{}-ref_data_names_cache'.format(project, build_system)
ref_data_names_map = cache.get(cache_key)
if not ref_data_names_map:
# cache expired so re-build the reference data names map; the map
# contains the ref_data_name of every treeherder *test* job for this project
ref_data_names_map = self._build_ref_data_names(project, build_system)
# update the cache
cache.set(cache_key, ref_data_names_map, SETA_REF_DATA_NAMES_CACHE_TIMEOUT)
# now check the JobPriority table against the list of valid runnable
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning('Job priority ({}) not found in accepted jobs list'.format(jp))
return jobs
def _build_ref_data_names(self, project, build_system):
'''
We want all reference data names for every task that runs on a specific project.
For example:
* Buildbot - "Windows 8 64-bit mozilla-inbound debug test web-platform-tests-1"
* TaskCluster = "test-linux64/opt-mochitest-webgl-e10s-1"
'''
ignored_jobs = []
ref_data_names = {}
runnable_jobs = list_runnable_jobs(project)['results']
for job in runnable_jobs:
# get testtype e.g. web-platform-tests-4
testtype = parse_testtype(
build_system_type=job['build_system_type'],
job_type_name=job['job_type_name'],
platform_option=job['platform_option'],
ref_data_name=job['ref_data_name']
)
if not valid_platform(job['platform']):
continue
if is_job_blacklisted(testtype):
ignored_jobs.append(job['ref_data_name'])
continue
key = unique_key(testtype=testtype,
buildtype=job['platform_option'],
platform=job['platform'])
if build_system == '*':
ref_data_names[key] = job['ref_data_name']
elif job['build_system_type'] == build_system:
ref_data_names[key] = job['ref_data_name']
for ref_data_name in sorted(ignored_jobs):
logger.info('Ignoring {}'.format(ref_data_name))
return ref_data_names
def _process(project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
jobs = []
# This map contains the ref_data_name of every Treeherder *test* job for this project
ref_data_names_map = build_ref_data_names(project, build_system)
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning('We did not find job priority ({}) in the list of accepted jobs'.format(jp))
return jobs
def _process(project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
jobs = []
# This map contains the ref_data_name of every Treeherder *test* job for this project
ref_data_names_map = build_ref_data_names(project, build_system)
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning(
'We did not find job priority ({}) in the list of accepted jobs'
.format(jp))
return jobs
def get_failures_fixed_by_commit():
""" Return all job failures annotated with "fixed by commit" grouped by reason given for annotation.
It returns a dictionary with a revision or bug ID as the key (bug ID is used for
intermittent failures and the revision is used for real failures). For SETA's purposes
we only care about revisions (real failures).
The failures for *real failures* will contain all jobs that have been starred as "fixed by commit".
Notice that the data does not tell you on which repository a root failure was fixed.
For instance, in the raw data you might see a reference to 9fa614d8310d which is a back out
and it is referenced by 12 starred jobs:
https://treeherder.mozilla.org/#/jobs?repo=autoland&filter-searchStr=android%20debug%20cpp&tochange=9fa614d8310db9aabe85cc3c3cff6281fe1edb0c
The raw data will show those 12 jobs.
The returned data will look like this:
{
"44d29bac3654": [
["android-4-0-armv7-api15", "opt", "android-lint"],
["android-4-0-armv7-api15", "opt", "android-api-15-gradle-dependencies"],
]
}
"""
failures = defaultdict(list)
option_collection_map = models.OptionCollection.objects.get_option_collection_map(
)
fixed_by_commit_data_set = models.JobNote.objects.filter(
failure_classification=2,
created__gt=timezone.now() - timedelta(days=SETA_FIXED_BY_COMMIT_DAYS),
text__isnull=False,
job__repository__name__in=SETA_FIXED_BY_COMMIT_REPOS).exclude(
job__signature__build_platform__in=SETA_UNSUPPORTED_PLATFORMS
).exclude(text="").select_related('job', 'job__signature',
'job__job_type')
# check if at least one fixed by commit job meets our requirements without populating queryset
if not fixed_by_commit_data_set.exists():
logger.warn("We couldn't find any fixed-by-commit jobs")
return failures
# now process the fixed by commit jobs in batches using django's queryset iterator
for job_note in fixed_by_commit_data_set.iterator():
# if we have http://hg.mozilla.org/rev/<rev> and <rev>, we will only use <rev>
revision_id = job_note.text.strip('/')
revision_id = revision_id.split('/')[-1]
# This prevents the empty string case and ignores bug ids
if not revision_id or len(revision_id) < 12:
continue
# We currently don't guarantee that text is actually a revision
# Even if not perfect the main idea is that a bunch of jobs were annotated with
# a unique identifier. The assumption is that the text is unique
#
# I've seen these values being used:
# * 12 char revision
# * 40 char revision
# * link to revision on hg
# * revisionA & revisionB
# * should be fixed by <revision>
# * bug id
#
# Note that if some jobs are annotated with the 12char revision and others with the
# 40char revision we will have two disjunct set of failures
#
# Some of this will be improved in https://bugzilla.mozilla.org/show_bug.cgi?id=1323536
try:
# check if jobtype is supported by SETA (see treeherder/seta/settings.py)
if job_note.job.signature.build_system_type != 'buildbot':
if not job_note.job.job_type.name.startswith(
tuple(SETA_SUPPORTED_TC_JOBTYPES)):
continue
testtype = parse_testtype(
build_system_type=job_note.job.signature.
build_system_type, # e.g. taskcluster
job_type_name=job_note.job.job_type.name, # e.g. Mochitest
platform_option=job_note.job.get_platform_option(
option_collection_map), # e.g. 'opt'
ref_data_name=job_note.job.signature.
name, # buildername or task label
)
if testtype:
if is_job_blacklisted(testtype):
continue
else:
logger.warning('We were unable to parse {}/{}'.format(
job_note.job.job_type.name, job_note.job.signature.name))
continue
# we now have a legit fixed-by-commit job failure
failures[revision_id].append(
unique_key(
testtype=testtype,
buildtype=job_note.job.get_platform_option(
option_collection_map), # e.g. 'opt'
platform=job_note.job.signature.build_platform))
except models.Job.DoesNotExist:
logger.warning('job_note {} has no job associated to it'.format(
job_note.id))
continue
logger.warn("Number of fixed_by_commit revisions: {}".format(
len(failures)))
return failures
def get_failures_fixed_by_commit():
""" Return all job failures annotated with "fixed by commit" grouped by reason given for annotation.
It returns a dictionary with a revision or bug ID as the key (bug ID is used for
intermittent failures and the revision is used for real failures). For SETA's purposes
we only care about revisions (real failures).
The failures for *real failures* will contain all jobs that have been starred as "fixed by commit".
Notice that the data does not tell you on which repository a root failure was fixed.
For instance, in the raw data you might see a reference to 9fa614d8310d which is a back out
and it is referenced by 12 starred jobs:
https://treeherder.mozilla.org/#/jobs?repo=autoland&filter-searchStr=android%20debug%20cpp&tochange=9fa614d8310db9aabe85cc3c3cff6281fe1edb0c
The raw data will show those 12 jobs.
The returned data will look like this:
{
"44d29bac3654": [
["android-4-0-armv7-api15", "opt", "android-lint"],
["android-4-0-armv7-api15", "opt", "android-api-15-gradle-dependencies"],
]
}
"""
failures = defaultdict(list)
option_collection_map = models.OptionCollection.objects.get_option_collection_map()
fixed_by_commit_data_set = models.JobNote.objects.filter(
failure_classification=2,
created__gt=timezone.now() - timedelta(days=SETA_FIXED_BY_COMMIT_DAYS),
text__isnull=False,
job__repository__name__in=SETA_FIXED_BY_COMMIT_REPOS
).exclude(
job__signature__build_platform__in=SETA_UNSUPPORTED_PLATFORMS
).exclude(
text=""
).select_related('job', 'job__signature', 'job__job_type')
# check if at least one fixed by commit job meets our requirements without populating queryset
if not fixed_by_commit_data_set.exists():
logger.warning("We couldn't find any fixed-by-commit jobs")
return failures
# now process the fixed by commit jobs in batches using django's queryset iterator
for job_note in fixed_by_commit_data_set.iterator():
# if we have http://hg.mozilla.org/rev/<rev> and <rev>, we will only use <rev>
revision_id = job_note.text.strip('/')
revision_id = revision_id.split('/')[-1]
# This prevents the empty string case and ignores bug ids
if not revision_id or len(revision_id) < 12:
continue
# We currently don't guarantee that text is actually a revision
# Even if not perfect the main idea is that a bunch of jobs were annotated with
# a unique identifier. The assumption is that the text is unique
#
# I've seen these values being used:
# * 12 char revision
# * 40 char revision
# * link to revision on hg
# * revisionA & revisionB
# * should be fixed by <revision>
# * bug id
#
# Note that if some jobs are annotated with the 12char revision and others with the
# 40char revision we will have two disjunct set of failures
#
# Some of this will be improved in https://bugzilla.mozilla.org/show_bug.cgi?id=1323536
try:
# check if jobtype is supported by SETA (see treeherder/seta/settings.py)
if job_note.job.signature.build_system_type != 'buildbot':
if not job_note.job.job_type.name.startswith(tuple(SETA_SUPPORTED_TC_JOBTYPES)):
continue
testtype = parse_testtype(
build_system_type=job_note.job.signature.build_system_type, # e.g. taskcluster
job_type_name=job_note.job.job_type.name, # e.g. Mochitest
platform_option=job_note.job.get_platform_option(option_collection_map), # e.g. 'opt'
ref_data_name=job_note.job.signature.name, # buildername or task label
)
if testtype:
if is_job_blacklisted(testtype):
continue
else:
logger.warning('We were unable to parse %s/%s',
job_note.job.job_type.name, job_note.job.signature.name)
continue
# we now have a legit fixed-by-commit job failure
failures[revision_id].append(unique_key(
testtype=testtype,
buildtype=job_note.job.get_platform_option(option_collection_map), # e.g. 'opt'
platform=job_note.job.signature.build_platform
))
except models.Job.DoesNotExist:
logger.warning('job_note %s has no job associated to it', job_note.id)
continue
logger.warning("Number of fixed_by_commit revisions: %s", len(failures))
return failures