def load_preseed(validate=False): """ Update JobPriority information from preseed.json""" logger.info("About to load preseed.json") preseed = preseed_data() if validate: logger.info("We are going to validate the values from preseed.json") ref_names = get_reference_data_names() for job in preseed: if validate: validate_preseed_entry(job, ref_names) logger.debug("Processing %s", (job["testtype"], job["buildtype"], job["platform"])) queryset = JobPriority.objects.all() for field in ('testtype', 'buildtype', 'platform'): if job[field] != '*': # The JobPriority table does not contain the raw # testtype value seen in the preseed.json file. We # must convert the job[field] value to the appropriate # value before performing the query. field_value = (convert_job_type_name_to_testtype(job[field]) if field == 'testtype' else job[field]) queryset = queryset.filter(**{field: field_value}) # Deal with the case where we have a new entry in preseed if not queryset: create_new_entry(job) else: # We can have wildcards, so loop on all returned values in data for jp in queryset: process_job_priority(jp, job) logger.debug("Finished")
def create_new_entry(job): if job['expiration_date'] == '*': job['expiration_date'] = THE_FUTURE logger.info("Adding a new job priority to the database: %s", job) testtype = convert_job_type_name_to_testtype(job['testtype']) JobPriority.objects.create( testtype=testtype, buildtype=job['buildtype'], platform=job['platform'], priority=job['priority'], expiration_date=job['expiration_date'], buildsystem=job['buildsystem'], )
def validate_preseed_entry(entry, ref_names): assert entry["testtype"] != "*" assert entry["buildtype"] != "*" assert entry["platform"] != "*" # We also support *, however, that was only useful with Buildbot assert entry["buildsystem"] == "taskcluster" # We support values different than *, however, it is not useful for preseed assert entry["expiration_date"] == "*" assert 1 <= entry["priority"] <= SETA_LOW_VALUE_PRIORITY # Collect potential matches potential_matches = [] for unique_identifier, ref_name in ref_names.items(): # XXX: Now that we have fuzzy build the term testtypes is not accurate if ref_name == entry["testtype"]: potential_matches.append(unique_identifier) assert len(potential_matches) > 0, Exception( "%s is not valid. Please check runnable_jobs.json from a Gecko decision task.", entry["testtype"], ) testtype = convert_job_type_name_to_testtype(entry["testtype"]) if not testtype: logger.warning( "Preseed.json entry testtype %s is not a valid task name:", entry["testtype"]) raise Exception( "preseed.json entry contains invalid testtype. Please check output above." ) unique_identifier = ( testtype, entry["buildtype"], entry["platform"], ) try: ref_names[unique_identifier] except KeyError: logger.warning("Preseed.json entry %s matches the following:", unique_identifier) logger.warning(potential_matches) raise Exception( "We failed to match your preseed.json entry. Please check output above." )
def parse_testtype(build_system_type, job_type_name, platform_option, ref_data_name): ''' Buildbot Taskcluster ----------- ----------- build_system_type buildbot taskcluster job_type_name Mochitest task label platform_option debug,opt,pgo debug,opt,pgo ref_data_name buildername task label OR signature hash ''' # XXX: Figure out how to ignore build, lint, etc. jobs # https://bugzilla.mozilla.org/show_bug.cgi?id=1318659 testtype = None if build_system_type == 'buildbot': # The testtype of builbot job can been found in 'ref_data_name' # like web-platform-tests-4 in "Ubuntu VM 12.04 x64 mozilla-inbound # opt test web-platform-tests-4" testtype = ref_data_name.split(' ')[-1] else: if job_type_name.startswith(tuple(SETA_SUPPORTED_TC_JOBTYPES)): # we should get "jittest-3" as testtype for a job_type_name like # test-linux64/debug-jittest-3 testtype = convert_job_type_name_to_testtype(job_type_name) return testtype