def load_preseed(validate=False):
""" Update JobPriority information from preseed.json"""
logger.info("About to load preseed.json")
preseed = preseed_data()
if validate:
logger.info("We are going to validate the values from preseed.json")
ref_names = get_reference_data_names()
for job in preseed:
if validate:
validate_preseed_entry(job, ref_names)
logger.debug("Processing %s",
(job["testtype"], job["buildtype"], job["platform"]))
queryset = JobPriority.objects.all()
for field in ('testtype', 'buildtype', 'platform'):
if job[field] != '*':
# The JobPriority table does not contain the raw
# testtype value seen in the preseed.json file. We
# must convert the job[field] value to the appropriate
# value before performing the query.
field_value = (convert_job_type_name_to_testtype(job[field])
if field == 'testtype' else job[field])
queryset = queryset.filter(**{field: field_value})
# Deal with the case where we have a new entry in preseed
if not queryset:
create_new_entry(job)
else:
# We can have wildcards, so loop on all returned values in data
for jp in queryset:
process_job_priority(jp, job)
logger.debug("Finished")
def create_new_entry(job):
if job['expiration_date'] == '*':
job['expiration_date'] = THE_FUTURE
logger.info("Adding a new job priority to the database: %s", job)
testtype = convert_job_type_name_to_testtype(job['testtype'])
JobPriority.objects.create(
testtype=testtype,
buildtype=job['buildtype'],
platform=job['platform'],
priority=job['priority'],
expiration_date=job['expiration_date'],
buildsystem=job['buildsystem'],
)
def validate_preseed_entry(entry, ref_names):
assert entry["testtype"] != "*"
assert entry["buildtype"] != "*"
assert entry["platform"] != "*"
# We also support *, however, that was only useful with Buildbot
assert entry["buildsystem"] == "taskcluster"
# We support values different than *, however, it is not useful for preseed
assert entry["expiration_date"] == "*"
assert 1 <= entry["priority"] <= SETA_LOW_VALUE_PRIORITY
# Collect potential matches
potential_matches = []
for unique_identifier, ref_name in ref_names.items():
# XXX: Now that we have fuzzy build the term testtypes is not accurate
if ref_name == entry["testtype"]:
potential_matches.append(unique_identifier)
assert len(potential_matches) > 0, Exception(
"%s is not valid. Please check runnable_jobs.json from a Gecko decision task.",
entry["testtype"],
)
testtype = convert_job_type_name_to_testtype(entry["testtype"])
if not testtype:
logger.warning(
"Preseed.json entry testtype %s is not a valid task name:",
entry["testtype"])
raise Exception(
"preseed.json entry contains invalid testtype. Please check output above."
)
unique_identifier = (
testtype,
entry["buildtype"],
entry["platform"],
)
try:
ref_names[unique_identifier]
except KeyError:
logger.warning("Preseed.json entry %s matches the following:",
unique_identifier)
logger.warning(potential_matches)
raise Exception(
"We failed to match your preseed.json entry. Please check output above."
)
def parse_testtype(build_system_type, job_type_name, platform_option, ref_data_name):
'''
Buildbot Taskcluster
----------- -----------
build_system_type buildbot taskcluster
job_type_name Mochitest task label
platform_option debug,opt,pgo debug,opt,pgo
ref_data_name buildername task label OR signature hash
'''
# XXX: Figure out how to ignore build, lint, etc. jobs
# https://bugzilla.mozilla.org/show_bug.cgi?id=1318659
testtype = None
if build_system_type == 'buildbot':
# The testtype of builbot job can been found in 'ref_data_name'
# like web-platform-tests-4 in "Ubuntu VM 12.04 x64 mozilla-inbound
# opt test web-platform-tests-4"
testtype = ref_data_name.split(' ')[-1]
else:
if job_type_name.startswith(tuple(SETA_SUPPORTED_TC_JOBTYPES)):
# we should get "jittest-3" as testtype for a job_type_name like
# test-linux64/debug-jittest-3
testtype = convert_job_type_name_to_testtype(job_type_name)
return testtype