def test_or(self):
merged = flags.FlagSet(flags.PASS) | flags.FlagSet(flags.LARGE)
self.assertEquals(flags.PASS, merged.status)
self.assertEquals(flags.LARGE, merged.attribute)
# If both operands has status, raise an AssertionError.
with self.assertRaises(AssertionError):
flags.FlagSet(flags.PASS) | flags.FlagSet(flags.NOT_SUPPORTED)
def _check_simple(self, expected, patterns):
self.assertEquals(
dict((key, flags.FlagSet(value))
for key, value in expected.iteritems()),
suite_runner_util.merge_expectation_map(
dict.fromkeys(expected, flags.FlagSet(flags.PASS)),
dict((key, flags.FlagSet(value))
for key, value in patterns.iteritems()),
flags.FlagSet(flags.PASS)))
def _make_suite_runner(self, name):
return suite_runner.SuiteRunnerBase(
name, {
'Class1#method1': flags.FlagSet(flags.PASS),
'Class1#method2': flags.FlagSet(flags.PASS),
'Class2#method1': flags.FlagSet(flags.PASS),
'Class2#method2': flags.FlagSet(flags.PASS),
},
config=SuiteRunConfigIntegrationTests.my_config()[name])
def test_defaults_applied(self):
result = _evaluate({'flags': flags.FlagSet(flags.PASS)},
defaults={
'bug': 'crbug.com/1234',
'flags': flags.FlagSet(flags.FAIL)
})
self.assertEquals('crbug.com/1234', result['bug'])
self.assertEquals(suite_runner_config._DEFAULT_OUTPUT_TIMEOUT,
result['deadline'])
self.assertEquals(flags.PASS, result['flags'].status)
def test_flat_suite_test_expectations(self):
result = _evaluate_test_expectations({'x': flags.FlagSet(flags.FLAKY)})
self.assertEqual(flags.FlagSet(flags.FLAKY), result['x'])
result = _evaluate_test_expectations({'*': flags.FlagSet(flags.FLAKY)})
self.assertEqual(flags.FlagSet(flags.FLAKY), result['*'])
# Only a simple '*' pattern is allowed.
# (Though this pattern still allows us to do a prefix match later, we
# disallow it.)
with self.assertRaisesRegexp(AssertionError, r'"x\*" is not allowed'):
_evaluate_test_expectations({'x*': flags.FlagSet(flags.PASS)})
# Only a simple '*' pattern is allowed.
# (This allows us to to a simple prefix match later)
with self.assertRaisesRegexp(AssertionError, r'"\*x" is not allowed'):
_evaluate_test_expectations({'*x': flags.FlagSet(flags.PASS)})
# A "class#method" style name is allowed.
result = _evaluate_test_expectations(
{'x#y': flags.FlagSet(flags.FLAKY)})
self.assertEqual(flags.FlagSet(flags.FLAKY), result['x#y'])
# Only one '#' is allowed.
with self.assertRaisesRegexp(AssertionError,
r'"x#y#z" is not allowed'):
_evaluate_test_expectations({'x#y#z': flags.FlagSet(flags.PASS)})
def test_eq_ne(self):
self.assertEquals(flags.FlagSet(flags.PASS), flags.FlagSet(flags.PASS))
self.assertNotEquals(flags.FlagSet(flags.PASS), flags.FlagSet(flags.FAIL))
self.assertEquals(flags.FlagSet(flags.PASS | flags.LARGE),
flags.FlagSet(flags.PASS | flags.LARGE))
self.assertNotEquals(flags.FlagSet(flags.PASS | flags.LARGE),
flags.FlagSet(flags.PASS))
# Can be compared to None.
self.assertNotEquals(flags.FlagSet(flags.PASS), None)
self.assertNotEquals(None, flags.FlagSet(flags.PASS))
def _evaluate_suite_test_expectations(raw_dict):
"""Flatten the (possibly-nested) suite_test_expectations dict."""
result = {}
for outer_name, outer_expectation in raw_dict.iteritems():
if isinstance(outer_expectation, flags.FlagSet):
result[outer_name] = (flags.FlagSet(
flags.PASS).override_with(outer_expectation))
continue
for inner_name, inner_expectation in outer_expectation.iteritems():
result['%s#%s' % (outer_name, inner_name)] = (flags.FlagSet(
flags.PASS).override_with(inner_expectation))
return result
def test_hierarchical_suite_test_expectations(self):
result = _evaluate_test_expectations(
{'x': {
'y': flags.FlagSet(flags.FLAKY)
}})
self.assertEqual(flags.FlagSet(flags.FLAKY), result['x#y'])
result = _evaluate_test_expectations(
{'x': {
'*': flags.FlagSet(flags.FLAKY)
}})
self.assertEqual(flags.FlagSet(flags.FLAKY), result['x#*'])
# Only a simple '*' pattern is allowed.
# (Though this pattern still allows us to do a prefix match later, we
# disallow it.)
with self.assertRaisesRegexp(AssertionError,
r'"x#y\*" is not allowed'):
_evaluate_test_expectations(
{'x': {
'y*': flags.FlagSet(flags.FLAKY)
}})
# Only a simple '*' pattern is allowed.
# (This allows us to use a simple prefix match later)
with self.assertRaisesRegexp(AssertionError,
r'"x#\*y" is not allowed'):
_evaluate_test_expectations(
{'x': {
'*y': flags.FlagSet(flags.FLAKY)
}})
# If there is an asterisk wildcard, it must be in the leaf.
# (This allows us to to a simple prefix match later)
with self.assertRaisesRegexp(AssertionError,
r'"\*" is not a valid name'):
_evaluate_test_expectations(
{'*': {
'x': flags.FlagSet(flags.FLAKY)
}})
# If there is an asterisk wildcard, it must be in the leaf.
# (This allows us to to a simple prefix match later)
with self.assertRaisesRegexp(AssertionError,
r'"\*" is not a valid name'):
_evaluate_test_expectations(
{'*': {
'*': flags.FlagSet(flags.FLAKY)
}})
# Only one '#' is allowed.
with self.assertRaisesRegexp(AssertionError,
r'"x#y#z" is not allowed'):
_evaluate_test_expectations(
{'x': {
'y#z': flags.FlagSet(flags.FLAKY)
}})
def test_property(self):
flagset = flags.FlagSet(flags.PASS)
self.assertEquals(flags.PASS, flagset.status)
self.assertEquals(0, flagset.attribute)
flagset = flags.FlagSet(flags.PASS | flags.LARGE)
self.assertEquals(flags.PASS, flagset.status)
self.assertEquals(flags.LARGE, flagset.attribute)
# Also, attribute only FlagSet is allowed.
flagset = flags.FlagSet(flags.LARGE)
self.assertEquals(0, flagset.status)
self.assertEquals(flags.LARGE, flagset.attribute)
def test_configured_to_fail_for_target(self):
result = _evaluate(
{'configurations': [{
'flags': flags.FlagSet(flags.FLAKY)
}]})
self.assertEquals(flags.FLAKY, result['flags'].status)
result = _evaluate({
'configurations': [{
'enable_if': False,
'flags': flags.FlagSet(flags.FLAKY)
}]
})
self.assertEquals(flags.PASS, result['flags'].status)
def _evaluate(raw_config, defaults=None):
"""Flatten the raw_config based on the configuration"""
_validate(raw_config)
result = {
'flags': flags.FlagSet(flags.PASS),
'bug': None,
'deadline': _DEFAULT_OUTPUT_TIMEOUT,
'test_order': {},
'suite_test_expectations': {},
'metadata': {},
}
if defaults:
# We need to make a deep copy so that we do not modify any dictionary or
# array data in place and affect the default values for subsequent use.
result.update(copy.deepcopy(defaults))
if not raw_config:
return result
# Merge configurations.
_merge_config(raw_config, result)
# Apply conditional configurations.
for configuration in raw_config.get('configurations', []):
if not configuration.get('enable_if', True):
continue
_merge_config(configuration, result)
return result
def default_run_configuration():
return _evaluate({
'configurations': [{
'enable_if': OPTIONS.weird(),
'flags': flags.FlagSet(flags.FLAKY),
}]
})
def _run(self, benchmark):
runner = art_test_runner.ArtTestRunner(
'901-perf', config={'flags': flags.FlagSet(flags.PASS)})
args = _prepare_integration_tests_args(100)
# We reuse scripts for integration tests in vm tests, and they expect
# that out/integration_tests exists. It is true if integration tests ran
# before calling perf_test.py, but not true for perf builders.
# Let's create it if it does not exist.
run_integration_tests.setup_output_directory(args.output_dir)
# Call setup_work_root() and prepare_to_run() iff source files
# to build tests exist. Perf builders do not have them, and can skip it.
# The builders have downloaded pre-built files.
if os.path.exists(os.path.join(runner.get_source_root(), 'etc')):
runner.setup_work_root()
runner.prepare_to_run([], args)
with contextlib.closing(
suite_runner.SuiteRunnerLogger(
runner.name, os.path.join(args.output_dir, runner.name),
False)) as logger:
runner.run_with_setup([benchmark], args, logger, None)
with open(logger.path) as f:
for line in f:
# Result line format is 'Benchmark <name>: <result> ms'.
match = _BENCHMARK_RESULT_RE.match(line)
if not match or match.group(1) != benchmark:
continue
return match.group(2)
raise InvalidResultError(benchmark)
def test_get_expectations_works_with_named_tests(self):
sb = scoreboard.Scoreboard(
'suite', {
'testPasses': flags.FlagSet(flags.PASS),
'testFails': flags.FlagSet(flags.FAIL),
'testTimesOut': flags.FlagSet(flags.TIMEOUT),
'testFlaky': flags.FlagSet(flags.FLAKY),
})
expectations = sb.get_expectations()
self.assertEquals(4, len(expectations))
self.assertEquals(scoreboard_constants.EXPECTED_PASS,
expectations['testPasses'])
self.assertEquals(scoreboard_constants.EXPECTED_FAIL,
expectations['testFails'])
self.assertEquals(scoreboard_constants.SKIPPED,
expectations['testTimesOut'])
self.assertEquals(scoreboard_constants.EXPECTED_FLAKE,
expectations['testFlaky'])
def _make_flaky_suite_configuration(*unused, **kwunused):
# We must return a new dictionary for every call.
return {
'flags': flags.FlagSet(flags.FLAKY),
'bug': None,
'metadata': {},
'deadline': 300,
'test_order': {},
'suite_test_expectations': {}
}
def test_flake_restart_pass(self):
expectations = {'flake': flags.FlagSet(flags.FLAKY)}
sb = scoreboard.Scoreboard('suite', expectations)
tests = ['flake']
self._register_tests(sb, tests)
sb.start(tests)
# Fail the test the first time.
actuals = {'flake': test_method_result.TestMethodResult.FAIL}
self._update_tests(sb, actuals)
results = {
'total': 1,
'completed': 1,
'get_flaky_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Restart the tests.
sb.restart(len(tests))
sb.start(tests)
# Pass the test the second time.
actuals = {'flake': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 2,
'restarts': 1,
'completed': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Verify finalized results.
sb.finalize()
results = {
'total': 2,
'restarts': 1,
'completed': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Note: the number of completed tests cannot be determined by simply
# adding up the individual results. This is because flaky tests that
# failed are neither passing nor failing nor skipped (but they most
# definitely were completed).
self.assertNotEqual(sb.completed, sb.passed + sb.failed + sb.skipped)
def test_flake_incomplete(self):
expectations = {'flake': flags.FlagSet(flags.FLAKY)}
sb = scoreboard.Scoreboard('suite', expectations)
tests = ['flake']
self._register_tests(sb, tests)
sb.start(tests)
# The test is never run.
self.assertEquals(self._test_counter[scoreboard_constants.INCOMPLETE],
0)
sb.finalize()
self.assertEquals(self._test_counter[scoreboard_constants.INCOMPLETE],
1)
def test_should_run(self):
# Check the default behavior.
instance = test_filter.TestRunFilter()
self.assertTrue(instance.should_run(flags.FlagSet(flags.PASS)))
self.assertTrue(instance.should_run(flags.FlagSet(flags.FLAKY)))
self.assertFalse(instance.should_run(flags.FlagSet(flags.FAIL)))
self.assertFalse(instance.should_run(flags.FlagSet(flags.TIMEOUT)))
self.assertFalse(
instance.should_run(flags.FlagSet(flags.NOT_SUPPORTED)))
self.assertFalse(
instance.should_run(flags.FlagSet(flags.PASS | flags.LARGE)))
instance = test_filter.TestRunFilter(include_fail=True)
self.assertTrue(instance.should_run(flags.FlagSet(flags.FAIL)))
instance = test_filter.TestRunFilter(include_large=True)
self.assertTrue(
instance.should_run(flags.FlagSet(flags.PASS | flags.LARGE)))
instance = test_filter.TestRunFilter(include_timeout=True)
self.assertTrue(instance.should_run(flags.FlagSet(flags.TIMEOUT)))
def _merge(base_expectation, override_expectation, default_expectation):
if override_expectation:
return default_expectation.override_with(override_expectation)
# We choose 'worse' status here. Considering;
# |base_expectation| should be either |PASS| or |FAIL|, because it is based
# on the original Android's test config. On the other hand,
# default_expectation is what we defined for each test suite. Specifically,
# it's |PASS| by default.
# It should make sense that a test case, which is marked |FAIL| by base
# but |PASS| by default, fails.
# On the other hand, if we annotate NOT_SUPPORTED or TIMEOUT for a test
# case, we do not want to run (by default) regardless of whether it is
# marked as PASS or FAIL by base expectation.
status = max(base_expectation.status, default_expectation.status)
return flags.FlagSet(status | default_expectation.attribute)
def test_flake_restart_fail(self):
expectations = {'flake': flags.FlagSet(flags.FLAKY)}
sb = scoreboard.Scoreboard('suite', expectations)
tests = ['flake']
self._register_tests(sb, tests)
sb.start(tests)
# Fail the test the first time.
actuals = {'flake': test_method_result.TestMethodResult.FAIL}
self._update_tests(sb, actuals)
results = {
'total': 1,
'completed': 1,
'get_flaky_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Restart the tests.
sb.restart(len(tests))
sb.start(tests)
# Pass the test the second time.
actuals = {'flake': test_method_result.TestMethodResult.FAIL}
self._update_tests(sb, actuals)
results = {
'total': 2,
'restarts': 1,
'completed': 2,
'get_flaky_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Verify finalized results.
sb.finalize()
results = {
'total': 2,
'completed': 2,
'failed': 1,
'restarts': 1,
'unexpected_failed': 1,
'get_unexpected_failing_tests': ['flake'],
'overall_status': scoreboard_constants.UNEXPECTED_FAIL,
}
self._check_scoreboard(sb, results)
def _read_test_config(path, on_bot, use_gpu, remote_host_type):
"""Reads the file, and eval() it with the test config context."""
if not os.path.exists(path):
return {}
with open(path) as stream:
content = stream.read()
test_context = {
'__builtin__': None, # Do not inherit the current context.
# Expectation flags.
'PASS': flags.FlagSet(flags.PASS),
'FLAKY': flags.FlagSet(flags.FLAKY),
'FAIL': flags.FlagSet(flags.FAIL),
'TIMEOUT': flags.FlagSet(flags.TIMEOUT),
'NOT_SUPPORTED': flags.FlagSet(flags.NOT_SUPPORTED),
'LARGE': flags.FlagSet(flags.LARGE),
# OPTIONS is commonly used for the conditions.
'OPTIONS': OPTIONS,
# Variables which can be used to check runtime configurations.
'ON_BOT': on_bot,
'USE_GPU': use_gpu,
'USE_NDK_DIRECT_EXECUTION': build_common.use_ndk_direct_execution(),
# Platform information of the machine on which the test runs for
# remote execution. If it is not the remote execution, all variables
# below will be False.
'ON_CYGWIN': remote_host_type == 'cygwin',
'ON_MAC': remote_host_type == 'mac',
'ON_CHROMEOS': remote_host_type == 'chromeos',
}
try:
raw_config = eval(content, test_context)
except Exception as e:
e.args = (e.args[0] + '\neval() failed: ' + path, ) + e.args[1:]
raise
try:
_validate(raw_config)
except Exception as e:
e.args = (e.args[0] + '\nValidation failed: ' + path, ) + e.args[1:]
raise
return raw_config
def __init__(self, name, expect, actual):
self._name = name
self._expect = flags.FlagSet(expect)
self._actual = actual
def test_blacklist(self):
expectations = {
'alpha': flags.FlagSet(flags.PASS),
'beta': flags.FlagSet(flags.PASS),
'gamma': flags.FlagSet(flags.PASS),
}
sb = scoreboard.Scoreboard('suite', expectations)
self._register_tests(sb, ['alpha', 'beta', 'gamma'])
# Run and pass just the first test.
sb.start(['alpha', 'beta', 'gamma'])
actuals = {'alpha': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 3,
'completed': 1,
'incompleted': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['alpha'],
'get_incomplete_tests': ['beta', 'gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# Restart.
sb.restart(2)
results = {
'total': 3,
'restarts': 1,
'completed': 1,
'incompleted': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['alpha'],
'get_incomplete_tests': ['beta', 'gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# Run the remaining two tests.
sb.start(['beta', 'gamma'])
actuals = {'beta': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 3,
'restarts': 1,
'completed': 2,
'incompleted': 1,
'passed': 2,
'expected_passed': 2,
'get_expected_passing_tests': ['alpha', 'beta'],
'get_incomplete_tests': ['gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# After this restart, 'gamma' will have been incomplete twice, so
# it should get added to the blacklist.
sb.restart(1)
results = {
'total': 3,
'restarts': 2,
'completed': 2,
'incompleted': 1,
'passed': 2,
'expected_passed': 2,
'get_expected_passing_tests': ['alpha', 'beta'],
'get_incomplete_tests': ['gamma'],
'get_incomplete_blacklist': ['gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# Finally run the last test. Now 'gamma' should no longer be in the
# blacklist since it ran.
sb.start(['gamma'])
actuals = {'gamma': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 3,
'completed': 3,
'passed': 3,
'expected_passed': 3,
'restarts': 2,
'get_expected_passing_tests': ['alpha', 'beta', 'gamma'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Verify all results.
sb.finalize()
results = {
'total': 3,
'completed': 3,
'passed': 3,
'expected_passed': 3,
'restarts': 2,
'get_expected_passing_tests': ['alpha', 'beta', 'gamma'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
def __init__(self, test_name, **kwargs):
super(UnittestRunner, self).__init__(
test_name, {UnittestRunner._TEST_NAME: flags.FlagSet(flags.PASS)},
**kwargs)
def test_simple_failing_test(self):
result = _evaluate({'flags': flags.FlagSet(flags.FAIL)})
self.assertEquals(flags.FAIL, result['flags'].status)
def test_simple_passing_test(self):
self.assertEquals(flags.PASS, _evaluate(None)['flags'].status)
self.assertEquals(flags.PASS, _evaluate({})['flags'].status)
self.assertEquals(
flags.PASS,
_evaluate({'flags': flags.FlagSet(flags.PASS)})['flags'].status)
def test_merge_expectation_map(self):
base_map = {
'c#test1': flags.FlagSet(flags.PASS),
'c#test2': flags.FlagSet(flags.FAIL),
}
# With no override expectations, the base expectations should be used.
self.assertEquals(
{
'c#test1': flags.FlagSet(flags.PASS),
'c#test2': flags.FlagSet(flags.FAIL),
},
suite_runner_util.merge_expectation_map(base_map, {},
flags.FlagSet(flags.PASS)))
# test1 should be overridden to FAIL, test2 should keep the base FAIL.
self.assertEquals(
{
'c#test1': flags.FlagSet(flags.FAIL),
'c#test2': flags.FlagSet(flags.FAIL),
},
suite_runner_util.merge_expectation_map(
base_map, {'c#test1': flags.FlagSet(flags.FAIL)},
flags.FlagSet(flags.PASS)))
# The pure expectation from the default expectation should end up in the
# output expectation map.
self.assertEquals(
{
'c#test1': flags.FlagSet(flags.FLAKY),
'c#test2': flags.FlagSet(flags.FAIL),
},
suite_runner_util.merge_expectation_map(base_map, {},
flags.FlagSet(
flags.FLAKY)))
# If the default expectation is TIMEOUT, all the tests inside should be too
# if no other test-level overrides are given
self.assertEquals(
{
'c#test1': flags.FlagSet(flags.FLAKY),
'c#test2': flags.FlagSet(flags.TIMEOUT),
},
suite_runner_util.merge_expectation_map(
base_map, {'c#test1': flags.FlagSet(flags.FLAKY)},
flags.FlagSet(flags.TIMEOUT)))
# A suite level FLAKY flag should cause all tests to be marked FLAKY,
# regardless of whether the base or override expectation is used.
self.assertEquals(
{
'c#test1': flags.FlagSet(flags.FAIL | flags.LARGE),
'c#test2': flags.FlagSet(flags.FAIL),
},
suite_runner_util.merge_expectation_map(
base_map, {'c#test1': flags.FlagSet(flags.FAIL | flags.LARGE)},
flags.FlagSet(flags.FLAKY)))
# A suite level LARGE flag should cause all tests to be marked LARGE,
# regardless of whether the base or override expectation is used.
self.assertEquals(
{
'c#test1': flags.FlagSet(flags.PASS | flags.LARGE),
'c#test2': flags.FlagSet(flags.PASS | flags.LARGE),
},
suite_runner_util.merge_expectation_map(
base_map, {'c#test2': flags.FlagSet(flags.PASS)},
flags.FlagSet(flags.PASS | flags.LARGE)))
with self.assertRaises(AssertionError):
# Raise an exception if suite_expectations contains an unknown test name.
suite_runner_util.merge_expectation_map(
base_map, {'c#test3': flags.FlagSet(flags.PASS)},
flags.FlagSet(flags.PASS))
def test_works_as_intended(self):
runner = self._make_suite_runner('dummy_suite_1')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.PASS),
'Class1#method2': flags.FlagSet(flags.PASS),
'Class2#method1': flags.FlagSet(flags.PASS),
'Class2#method2': flags.FlagSet(flags.PASS),
}, runner.expectation_map)
self.assertEquals(None, runner.bug)
runner = self._make_suite_runner('dummy_suite_2')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.PASS),
'Class1#method2': flags.FlagSet(flags.PASS),
'Class2#method1': flags.FlagSet(flags.PASS),
'Class2#method2': flags.FlagSet(flags.PASS),
}, runner.expectation_map)
self.assertEquals(None, runner.bug)
runner = self._make_suite_runner('dummy_suite_3')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.FAIL),
'Class1#method2': flags.FlagSet(flags.FAIL),
'Class2#method1': flags.FlagSet(flags.FAIL),
'Class2#method2': flags.FlagSet(flags.FAIL),
}, runner.expectation_map)
self.assertEquals('crbug.com/123123', runner.bug)
runner = self._make_suite_runner('dummy_suite_4')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.LARGE | flags.FAIL),
'Class1#method2': flags.FlagSet(flags.LARGE | flags.FLAKY),
'Class2#method1': flags.FlagSet(flags.LARGE | flags.TIMEOUT),
'Class2#method2': flags.FlagSet(flags.LARGE | flags.PASS),
}, runner.expectation_map)
self.assertEquals(None, runner.bug)
self.assertEquals(['priMethod', 'abcMethod', 'xyzMethod'],
runner.apply_test_ordering(
['xyzMethod', 'abcMethod', 'priMethod']))
def read_test_list(path):
"""Reads a list of test methods from file, and returns an expectation map."""
with open(path) as stream:
data = stream.read()
return dict.fromkeys(data.splitlines(), flags.FlagSet(flags.PASS))
class SuiteRunConfigIntegrationTests(unittest.TestCase):
"""Uses the module interface as intended."""
# This is the configuration the tests will use:
my_config = staticmethod(
suite_runner_config.make_suite_run_configs(
lambda: {
suite_runner_config.SUITE_DEFAULTS: {
'flags': flags.FlagSet(flags.PASS),
'deadline': 60,
},
'dummy_suite_1': None,
'dummy_suite_2': {},
'dummy_suite_3': {
'flags': flags.FlagSet(flags.FAIL),
'bug': 'crbug.com/123123',
},
'dummy_suite_4': {
'flags':
flags.FlagSet(flags.LARGE),
'configurations': [{
'test_order':
collections.OrderedDict([('priMethod', -1)]),
'suite_test_expectations': {
'Class1': {
'method1': flags.FlagSet(flags.FAIL),
'method2': flags.FlagSet(flags.FLAKY),
},
'Class2#method1': flags.FlagSet(flags.TIMEOUT),
},
}],
},
}))
def setUp(self):
OPTIONS.parse([])
def _make_suite_runner(self, name):
return suite_runner.SuiteRunnerBase(
name, {
'Class1#method1': flags.FlagSet(flags.PASS),
'Class1#method2': flags.FlagSet(flags.PASS),
'Class2#method1': flags.FlagSet(flags.PASS),
'Class2#method2': flags.FlagSet(flags.PASS),
},
config=SuiteRunConfigIntegrationTests.my_config()[name])
def test_works_as_intended(self):
runner = self._make_suite_runner('dummy_suite_1')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.PASS),
'Class1#method2': flags.FlagSet(flags.PASS),
'Class2#method1': flags.FlagSet(flags.PASS),
'Class2#method2': flags.FlagSet(flags.PASS),
}, runner.expectation_map)
self.assertEquals(None, runner.bug)
runner = self._make_suite_runner('dummy_suite_2')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.PASS),
'Class1#method2': flags.FlagSet(flags.PASS),
'Class2#method1': flags.FlagSet(flags.PASS),
'Class2#method2': flags.FlagSet(flags.PASS),
}, runner.expectation_map)
self.assertEquals(None, runner.bug)
runner = self._make_suite_runner('dummy_suite_3')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.FAIL),
'Class1#method2': flags.FlagSet(flags.FAIL),
'Class2#method1': flags.FlagSet(flags.FAIL),
'Class2#method2': flags.FlagSet(flags.FAIL),
}, runner.expectation_map)
self.assertEquals('crbug.com/123123', runner.bug)
runner = self._make_suite_runner('dummy_suite_4')
self.assertEquals(60, runner.deadline)
self.assertEquals(
{
'Class1#method1': flags.FlagSet(flags.LARGE | flags.FAIL),
'Class1#method2': flags.FlagSet(flags.LARGE | flags.FLAKY),
'Class2#method1': flags.FlagSet(flags.LARGE | flags.TIMEOUT),
'Class2#method2': flags.FlagSet(flags.LARGE | flags.PASS),
}, runner.expectation_map)
self.assertEquals(None, runner.bug)
self.assertEquals(['priMethod', 'abcMethod', 'xyzMethod'],
runner.apply_test_ordering(
['xyzMethod', 'abcMethod', 'priMethod']))
