def testDisabledStringOnFunction(self):
@decorators.Disabled('bar')
def Sum():
return 1 + 1
self.assertEquals({'bar'}, decorators.GetDisabledAttributes(Sum))
@decorators.Disabled('bar')
@decorators.Disabled('baz')
@decorators.Disabled('bart', 'baz')
def Product():
return 1 * 1
self.assertEquals({'bar', 'bart', 'baz'},
decorators.GetDisabledAttributes(Product))
def testDisabledStringOnClass(self):
@decorators.Disabled('windshield')
class Ford(object):
pass
self.assertEquals({'windshield'},
decorators.GetDisabledAttributes(Ford))
@decorators.Disabled('windows', 'Drive')
@decorators.Disabled('wheel')
@decorators.Disabled('windows')
class Honda(object):
pass
self.assertEquals({'wheel', 'Drive', 'windows'},
decorators.GetDisabledAttributes(Honda))
def check_decorators(benchmarks):
for benchmark in benchmarks:
if (decorators.GetDisabledAttributes(benchmark)
or decorators.GetEnabledAttributes(benchmark)):
raise Exception(
'Disabling or enabling telemetry benchmark with decorator detected. '
'Please use StoryExpectations instead. Contact rnephew@ for more '
'information. \nBenchmark: %s' % benchmark.Name())
def testDisabledStringOnMethod(self):
class Ford(object):
@decorators.Disabled('windshield')
def Drive(self):
pass
self.assertEquals({'windshield'},
decorators.GetDisabledAttributes(Ford().Drive))
class Honda(object):
@decorators.Disabled('windows', 'Drive')
@decorators.Disabled('wheel')
@decorators.Disabled('windows')
def Drive(self):
pass
self.assertEquals({'wheel', 'Drive', 'windows'},
decorators.GetDisabledAttributes(Honda().Drive))
self.assertEquals({'windshield'},
decorators.GetDisabledAttributes(Ford().Drive))
class Accord(Honda):
def Drive(self):
pass
class Explorer(Ford):
pass
self.assertEquals({'wheel', 'Drive', 'windows'},
decorators.GetDisabledAttributes(Honda().Drive))
self.assertEquals({'windshield'},
decorators.GetDisabledAttributes(Ford().Drive))
self.assertEquals({'windshield'},
decorators.GetDisabledAttributes(Explorer().Drive))
self.assertFalse(decorators.GetDisabledAttributes(Accord().Drive))
def get_all_benchmarks_metadata(metadata):
benchmark_list = current_benchmarks()
for benchmark in benchmark_list:
disabled = 'all' in decorators.GetDisabledAttributes(benchmark)
emails = decorators.GetEmails(benchmark)
if emails:
emails = ', '.join(emails)
metadata[benchmark.Name()] = BenchmarkMetadata(
emails, decorators.GetComponent(benchmark), disabled)
return metadata
def get_all_benchmarks_metadata(metadata):
benchmark_list = current_benchmarks()
for benchmark in benchmark_list:
exp = benchmark().GetExpectations()
disabled = 'all' in decorators.GetDisabledAttributes(benchmark) or any(
any(isinstance(condition, expectations.ALL.__class__)
for condition in conditions)
for (conditions, _) in exp.disabled_platforms)
emails = decorators.GetEmails(benchmark)
if emails:
emails = ', '.join(emails)
metadata[benchmark.Name()] = BenchmarkMetadata(
emails, decorators.GetComponent(benchmark), disabled)
return metadata
def IsBenchmarkDisabledOnTrybotPlatform(cls, benchmark_class, trybot_name):
"""Return whether benchmark will be disabled on trybot platform.
Note that we cannot tell with certainty whether the benchmark will be
disabled on the trybot platform since the disable logic in ShouldDisable()
can be very dynamic and can only be verified on the trybot server platform.
We are biased on the side of enabling the benchmark, and attempt to
early discover whether the benchmark will be disabled as our best.
It should never be the case that the benchmark will be enabled on the test
platform but this method returns True.
Returns:
A tuple (is_benchmark_disabled, reason) whereas |is_benchmark_disabled| is
a boolean that tells whether we are sure that the benchmark will be
disabled, and |reason| is a string that shows the reason why we think the
benchmark is disabled for sure.
"""
benchmark_name = benchmark_class.Name()
benchmark_disabled_strings = decorators.GetDisabledAttributes(
benchmark_class)
if 'all' in benchmark_disabled_strings:
return True, 'Benchmark %s is disabled on all platform.' % benchmark_name
if trybot_name == 'all':
return False, ''
trybot_platform = _GetBotPlatformFromTrybotName(trybot_name)
if trybot_platform in benchmark_disabled_strings:
return True, (
"Benchmark %s is disabled on %s, and trybot's platform is %s."
% (benchmark_name, ', '.join(benchmark_disabled_strings),
trybot_platform))
benchmark_enabled_strings = decorators.GetEnabledAttributes(
benchmark_class)
if (benchmark_enabled_strings
and trybot_platform not in benchmark_enabled_strings
and 'all' not in benchmark_enabled_strings):
return True, (
"Benchmark %s is only enabled on %s, and trybot's platform is %s."
% (benchmark_name, ', '.join(benchmark_enabled_strings),
trybot_platform))
if benchmark_class.ShouldDisable != benchmark.Benchmark.ShouldDisable:
logging.warning(
'Benchmark %s has ShouldDisable() method defined. If your trybot run '
'does not produce any results, it is possible that the benchmark '
'is disabled on the target trybot platform.', benchmark_name)
return False, ''
def ShouldBenchmarkBeScheduled(benchmark, platform):
disabled_tags = decorators.GetDisabledAttributes(benchmark)
enabled_tags = decorators.GetEnabledAttributes(benchmark)
# Don't run benchmarks which are disabled on all platforms.
if 'all' in disabled_tags:
return False
# If we're not on android, don't run mobile benchmarks.
if platform != 'android' and 'android' in enabled_tags:
return False
# If we're on android, don't run benchmarks disabled on mobile
if platform == 'android' and 'android' in disabled_tags:
return False
return True
def runTest(self):
all_benchmarks = _GetAllPerfBenchmarks()
names_to_benchmarks = defaultdict(list)
for b in all_benchmarks:
names_to_benchmarks[b.Name()] = b
for n, bench in names_to_benchmarks.items():
if 'mobile' in n:
enabled_tags = decorators.GetEnabledAttributes(bench)
disabled_tags = decorators.GetDisabledAttributes(bench)
self.assertTrue(
'all' in disabled_tags or 'android' in enabled_tags,
','.join([
str(bench),
bench.Name(),
str(disabled_tags),
str(enabled_tags)
]))
