def _process(self, project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
if not job_priorities:
raise SetaError("Call docker-compose run backend ./manage.py initialize_seta")
jobs = []
ref_data_names_map = get_reference_data_names(project, build_system)
# now check the JobPriority table against the list of valid runnable
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning(
'Job priority key %s for (%s) not found in accepted jobs list', key, jp
)
return jobs
def _process(self, project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
jobs = []
# we cache the reference data names in order to reduce API calls
cache_key = '{}-{}-ref_data_names_cache'.format(project, build_system)
ref_data_names_map = cache.get(cache_key)
if not ref_data_names_map:
# cache expired so re-build the reference data names map; the map
# contains the ref_data_name of every treeherder *test* job for this project
ref_data_names_map = self._build_ref_data_names(project, build_system)
# update the cache
cache.set(cache_key, ref_data_names_map, SETA_REF_DATA_NAMES_CACHE_TIMEOUT)
# now check the JobPriority table against the list of valid runnable
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning('Job priority ({}) not found in accepted jobs list'.format(jp))
return jobs
def _sanitize_data(runnable_jobs_data):
"""We receive data from runnable jobs api and return the sanitized data that meets our needs.
This is a loop to remove duplicates (including buildsystem -> * transformations if needed)
By doing this, it allows us to have a single database query
It returns sanitized_list which will contain a subset which excludes:
* jobs that don't specify the platform
* jobs that don't specify the testtype
* if the job appears again, we replace build_system_type with '*'. By doing so, if a job appears
under both 'buildbot' and 'taskcluster', its build_system_type will be '*'
"""
job_build_system_type = {}
sanitized_list = []
for job in runnable_jobs_data:
if not valid_platform(job['platform']):
logger.debug('Invalid platform %s', job['platform'])
continue
testtype = parse_testtype(
build_system_type=job['build_system_type'],
job_type_name=job['job_type_name'],
platform_option=job['platform_option'],
ref_data_name=job['ref_data_name'],
)
if not testtype:
continue
# NOTE: This is *all* the data we need from the runnable API
new_job = {
'build_system_type':
job['build_system_type'], # e.g. {buildbot,taskcluster,*}
'platform': job['platform'], # e.g. windows8-64
'platform_option': job['platform_option'], # e.g. {opt,debug}
'testtype': testtype, # e.g. web-platform-tests-1
}
key = _unique_key(new_job)
# Let's build a map of all the jobs and if duplicated change the build_system_type to *
if key not in job_build_system_type:
job_build_system_type[key] = job['build_system_type']
sanitized_list.append(new_job)
elif new_job['build_system_type'] != job_build_system_type[key]:
new_job['build_system_type'] = job_build_system_type[key]
# This will *replace* the previous build system type with '*'
# This guarantees that we don't have duplicates
sanitized_list[sanitized_list.index(
new_job)]['build_system_type'] = '*'
return sanitized_list
def _sanitize_data(runnable_jobs_data):
"""We receive data from runnable jobs api and return the sanitized data that meets our needs.
This is a loop to remove duplicates (including buildsystem -> * transformations if needed)
By doing this, it allows us to have a single database query
It returns sanitized_list which will contain a subset which excludes:
* jobs that don't specify the platform
* jobs that don't specify the testtype
* if the job appears again, we replace build_system_type with '*'. By doing so, if a job appears
under both 'buildbot' and 'taskcluster', its build_system_type will be '*'
"""
job_build_system_type = {}
sanitized_list = []
for job in runnable_jobs_data:
if not valid_platform(job['platform']):
logger.info('Invalid platform %s', job['platform'])
continue
testtype = parse_testtype(
build_system_type=job['build_system_type'],
job_type_name=job['job_type_name'],
platform_option=job['platform_option'],
ref_data_name=job['ref_data_name']
)
if not testtype:
continue
# NOTE: This is *all* the data we need from the runnable API
new_job = {
'build_system_type': job['build_system_type'], # e.g. {buildbot,taskcluster,*}
'platform': job['platform'], # e.g. windows8-64
'platform_option': job['platform_option'], # e.g. {opt,debug}
'testtype': testtype, # e.g. web-platform-tests-1
}
key = _unique_key(new_job)
# Let's build a map of all the jobs and if duplicated change the build_system_type to *
if key not in job_build_system_type:
job_build_system_type[key] = job['build_system_type']
sanitized_list.append(new_job)
elif new_job['build_system_type'] != job_build_system_type[key]:
new_job['build_system_type'] = job_build_system_type[key]
# This will *replace* the previous build system type with '*'
# This guarantees that we don't have duplicates
sanitized_list[sanitized_list.index(new_job)]['build_system_type'] = '*'
return sanitized_list
def _build_ref_data_names(self, project, build_system):
'''
We want all reference data names for every task that runs on a specific project.
For example:
* Buildbot - "Windows 8 64-bit mozilla-inbound debug test web-platform-tests-1"
* TaskCluster = "test-linux64/opt-mochitest-webgl-e10s-1"
'''
ignored_jobs = []
ref_data_names = {}
runnable_jobs = list_runnable_jobs(project)['results']
for job in runnable_jobs:
# get testtype e.g. web-platform-tests-4
testtype = parse_testtype(
build_system_type=job['build_system_type'],
job_type_name=job['job_type_name'],
platform_option=job['platform_option'],
ref_data_name=job['ref_data_name']
)
if not valid_platform(job['platform']):
continue
if is_job_blacklisted(testtype):
ignored_jobs.append(job['ref_data_name'])
continue
key = unique_key(testtype=testtype,
buildtype=job['platform_option'],
platform=job['platform'])
if build_system == '*':
ref_data_names[key] = job['ref_data_name']
elif job['build_system_type'] == build_system:
ref_data_names[key] = job['ref_data_name']
for ref_data_name in sorted(ignored_jobs):
logger.info('Ignoring {}'.format(ref_data_name))
return ref_data_names
def _process(project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
jobs = []
# This map contains the ref_data_name of every Treeherder *test* job for this project
ref_data_names_map = build_ref_data_names(project, build_system)
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning('We did not find job priority ({}) in the list of accepted jobs'.format(jp))
return jobs
def _process(project, build_system, job_priorities):
'''Return list of ref_data_name for job_priorities'''
jobs = []
# This map contains the ref_data_name of every Treeherder *test* job for this project
ref_data_names_map = build_ref_data_names(project, build_system)
for jp in job_priorities:
# if this JobPriority entry is no longer supported in SETA then ignore it
if not valid_platform(jp.platform):
continue
if is_job_blacklisted(jp.testtype):
continue
key = jp.unique_identifier()
if key in ref_data_names_map:
# e.g. desktop-test-linux64-pgo/opt-reftest-13 or builder name
jobs.append(ref_data_names_map[key])
else:
logger.warning(
'We did not find job priority ({}) in the list of accepted jobs'
.format(jp))
return jobs
