def filter(task):
if task.kind != "test":
return True
build_platform = task.attributes.get("build_platform")
build_type = task.attributes.get("build_type")
shippable = task.attributes.get("shippable", False)
if not build_platform or not build_type:
return True
family = platform_family(build_platform)
# We need to know whether this test is against a "regular" opt build
# (which is to say, not shippable, asan, tsan, or any other opt build
# with other properties). There's no positive test for this, so we have to
# do it somewhat hackily. Android doesn't have variants other than shippable
# so it is pretty straightforward to check for. Other platforms have many
# variants, but none of the regular opt builds we're looking for have a "-"
# in their platform name, so this works (for now).
is_regular_opt = (family == "android"
and not shippable) or "-" not in build_platform
if build_type != "opt" or not is_regular_opt:
return True
return False
def set_schedules_optimization(config, jobs):
"""Set the `skip-unless-affected` optimization based on the build platform."""
for job in jobs:
# don't add skip-unless-schedules if there's already a when defined
if "when" in job:
yield job
continue
build_platform = job["attributes"]["build_platform"]
job.setdefault("optimization", {"build": [platform_family(build_platform)]})
yield job
def set_schedules_optimization(config, jobs):
"""Set the `skip-unless-affected` optimization based on the build platform."""
for job in jobs:
# don't add skip-unless-schedules if there's already a when defined
if 'when' in job:
yield job
continue
build_platform = job['attributes']['build_platform']
job.setdefault('optimization',
{'build': [platform_family(build_platform)]})
yield job
def set_schedules_optimization(config, jobs):
"""Set the `skip-unless-affected` optimization based on the build platform."""
for job in jobs:
# don't add skip-unless-schedules if there's already a when defined
if 'when' in job:
yield job
continue
build_platform = job['attributes']['build_platform']
job.setdefault('optimization',
{'skip-unless-schedules': [platform_family(build_platform)]})
yield job
def make_job_description(config, tests):
"""Convert *test* descriptions to *job* descriptions (input to
taskgraph.transforms.job)"""
for test in tests:
label = '{}-{}-{}'.format(config.kind, test['test-platform'],
test['test-name'])
if test['chunks'] > 1:
label += '-{}'.format(test['this-chunk'])
build_label = test['build-label']
try_name = test['try-name']
if test['suite'] == 'talos':
attr_try_name = 'talos_try_name'
else:
attr_try_name = 'unittest_try_name'
attr_build_platform, attr_build_type = test['build-platform'].split(
'/', 1)
attributes = test.get('attributes', {})
attributes.update({
'build_platform': attr_build_platform,
'build_type': attr_build_type,
'test_platform': test['test-platform'],
'test_chunk': str(test['this-chunk']),
attr_try_name: try_name,
})
jobdesc = {}
name = '{}-{}'.format(test['test-platform'], test['test-name'])
jobdesc['name'] = name
jobdesc['label'] = label
jobdesc['description'] = test['description']
jobdesc['attributes'] = attributes
jobdesc['dependencies'] = {'build': build_label}
if test['mozharness']['requires-signed-builds'] is True:
jobdesc['dependencies']['build-signing'] = test[
'build-signing-label']
jobdesc['expires-after'] = test['expires-after']
jobdesc['routes'] = []
jobdesc['run-on-projects'] = test['run-on-projects']
jobdesc['scopes'] = []
jobdesc['tags'] = test.get('tags', {})
jobdesc['extra'] = {
'chunks': {
'current': test['this-chunk'],
'total': test['chunks'],
},
'suite': {
'name': attributes['unittest_suite'],
'flavor': attributes['unittest_flavor'],
},
}
jobdesc['treeherder'] = {
'symbol':
test['treeherder-symbol'],
'kind':
'test',
'tier':
test['tier'],
'platform':
test.get('treeherder-machine-platform', test['build-platform']),
}
if test.get('when'):
jobdesc['when'] = test['when']
else:
schedules = [
attributes['unittest_suite'],
platform_family(test['build-platform'])
]
if config.params['project'] != 'try':
# for non-try branches, include SETA
jobdesc['optimization'] = {
'skip-unless-schedules-or-seta': schedules
}
else:
# otherwise just use skip-unless-schedules
jobdesc['optimization'] = {'skip-unless-schedules': schedules}
run = jobdesc['run'] = {}
run['using'] = 'mozharness-test'
run['test'] = test
jobdesc['worker-type'] = test.pop('worker-type')
yield jobdesc
def make_job_description(config, tests):
"""Convert *test* descriptions to *job* descriptions (input to
taskgraph.transforms.job)"""
for test in tests:
label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
if test['chunks'] > 1:
label += '-{}'.format(test['this-chunk'])
build_label = test['build-label']
try_name = test['try-name']
if test['suite'] == 'talos':
attr_try_name = 'talos_try_name'
else:
attr_try_name = 'unittest_try_name'
attr_build_platform, attr_build_type = test['build-platform'].split('/', 1)
attributes = test.get('attributes', {})
attributes.update({
'build_platform': attr_build_platform,
'build_type': attr_build_type,
'test_platform': test['test-platform'],
'test_chunk': str(test['this-chunk']),
attr_try_name: try_name,
})
jobdesc = {}
name = '{}-{}'.format(test['test-platform'], test['test-name'])
jobdesc['name'] = name
jobdesc['label'] = label
jobdesc['description'] = test['description']
jobdesc['attributes'] = attributes
jobdesc['dependencies'] = {'build': build_label}
jobdesc['job-from'] = test['job-from']
if test['mozharness']['requires-signed-builds'] is True:
jobdesc['dependencies']['build-signing'] = test['build-signing-label']
jobdesc['expires-after'] = test['expires-after']
jobdesc['routes'] = []
jobdesc['run-on-projects'] = test['run-on-projects']
jobdesc['scopes'] = []
jobdesc['tags'] = test.get('tags', {})
jobdesc['extra'] = {
'chunks': {
'current': test['this-chunk'],
'total': test['chunks'],
},
'suite': {
'name': attributes['unittest_suite'],
'flavor': attributes['unittest_flavor'],
},
}
jobdesc['treeherder'] = {
'symbol': test['treeherder-symbol'],
'kind': 'test',
'tier': test['tier'],
'platform': test.get('treeherder-machine-platform', test['build-platform']),
}
suite = test.get('schedules-component', attributes['unittest_suite'])
if suite in INCLUSIVE_COMPONENTS:
# if this is an "inclusive" test, then all files which might
# cause it to run are annotated with SCHEDULES in moz.build,
# so do not include the platform or any other components here
schedules = [suite]
else:
schedules = [suite, platform_family(test['build-platform'])]
if test.get('when'):
jobdesc['when'] = test['when']
elif 'optimization' in test:
jobdesc['optimization'] = test['optimization']
elif not config.params.is_try() and suite not in INCLUSIVE_COMPONENTS:
# for non-try branches and non-inclusive suites, include SETA
jobdesc['optimization'] = {'skip-unless-schedules-or-seta': schedules}
else:
# otherwise just use skip-unless-schedules
jobdesc['optimization'] = {'skip-unless-schedules': schedules}
run = jobdesc['run'] = {}
run['using'] = 'mozharness-test'
run['test'] = test
jobdesc['worker-type'] = test.pop('worker-type')
yield jobdesc
def make_job_description(config, tests):
"""Convert *test* descriptions to *job* descriptions (input to
taskgraph.transforms.job)"""
for test in tests:
label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
if test['chunks'] > 1:
label += '-{}'.format(test['this-chunk'])
build_label = test['build-label']
try_name = test['try-name']
if test['suite'] == 'talos':
attr_try_name = 'talos_try_name'
else:
attr_try_name = 'unittest_try_name'
attr_build_platform, attr_build_type = test['build-platform'].split('/', 1)
attributes = test.get('attributes', {})
attributes.update({
'build_platform': attr_build_platform,
'build_type': attr_build_type,
'test_platform': test['test-platform'],
'test_chunk': str(test['this-chunk']),
attr_try_name: try_name,
})
jobdesc = {}
name = '{}-{}'.format(test['test-platform'], test['test-name'])
jobdesc['name'] = name
jobdesc['label'] = label
jobdesc['description'] = test['description']
jobdesc['attributes'] = attributes
jobdesc['dependencies'] = {'build': build_label}
jobdesc['job-from'] = test['job-from']
if test['mozharness']['requires-signed-builds'] is True:
jobdesc['dependencies']['build-signing'] = test['build-signing-label']
jobdesc['expires-after'] = test['expires-after']
jobdesc['routes'] = []
jobdesc['run-on-projects'] = test['run-on-projects']
jobdesc['scopes'] = []
jobdesc['tags'] = test.get('tags', {})
jobdesc['extra'] = {
'chunks': {
'current': test['this-chunk'],
'total': test['chunks'],
},
'suite': {
'name': attributes['unittest_suite'],
'flavor': attributes['unittest_flavor'],
},
}
jobdesc['treeherder'] = {
'symbol': test['treeherder-symbol'],
'kind': 'test',
'tier': test['tier'],
'platform': test.get('treeherder-machine-platform', test['build-platform']),
}
suite = test.get('schedules-component', attributes['unittest_suite'])
if suite in INCLUSIVE_COMPONENTS:
# if this is an "inclusive" test, then all files which might
# cause it to run are annotated with SCHEDULES in moz.build,
# so do not include the platform or any other components here
schedules = [suite]
else:
schedules = [suite, platform_family(test['build-platform'])]
if test.get('when'):
jobdesc['when'] = test['when']
elif config.params['project'] != 'try':
# for non-try branches, include SETA
jobdesc['optimization'] = {'skip-unless-schedules-or-seta': schedules}
else:
# otherwise just use skip-unless-schedules
jobdesc['optimization'] = {'skip-unless-schedules': schedules}
run = jobdesc['run'] = {}
run['using'] = 'mozharness-test'
run['test'] = test
jobdesc['worker-type'] = test.pop('worker-type')
yield jobdesc
