def __init__(self, name, base_expectation_map, config=None): merged_config = suite_runner_config.default_run_configuration() if config: merged_config.update(config) self._lock = threading.Lock() self._name = name self._terminated = False self._deadline = merged_config.pop('deadline') self._bug = merged_config.pop('bug') self._metadata = merged_config.pop('metadata') self._test_order = merged_config.pop('test_order') default_expectation = merged_config.pop('flags') override_expectation_map = merged_config.pop('suite_test_expectations') assert not merged_config, ('Unexpected keyword arguments %s' % merged_config.keys()) expectation_map = suite_runner_util.merge_expectation_map( base_expectation_map, override_expectation_map, default_expectation) self._expectation_map = expectation_map # These will be set up later, in prepare_to_run(), and run_subprocess(). self._args = None self._logger = None self._subprocess = None self._user_data_dir = None
def _check_simple(self, expected, patterns): self.assertEquals( dict((key, flags.FlagSet(value)) for key, value in expected.iteritems()), suite_runner_util.merge_expectation_map( dict.fromkeys(expected, flags.FlagSet(flags.PASS)), dict((key, flags.FlagSet(value)) for key, value in patterns.iteritems()), flags.FlagSet(flags.PASS)))
def test_merge_expectation_map(self): base_map = { 'c#test1': flags.FlagSet(flags.PASS), 'c#test2': flags.FlagSet(flags.FAIL), } # With no override expectations, the base expectations should be used. self.assertEquals( { 'c#test1': flags.FlagSet(flags.PASS), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map(base_map, {}, flags.FlagSet(flags.PASS))) # test1 should be overridden to FAIL, test2 should keep the base FAIL. self.assertEquals( { 'c#test1': flags.FlagSet(flags.FAIL), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map( base_map, {'c#test1': flags.FlagSet(flags.FAIL)}, flags.FlagSet(flags.PASS))) # The pure expectation from the default expectation should end up in the # output expectation map. self.assertEquals( { 'c#test1': flags.FlagSet(flags.FLAKY), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map(base_map, {}, flags.FlagSet( flags.FLAKY))) # If the default expectation is TIMEOUT, all the tests inside should be too # if no other test-level overrides are given self.assertEquals( { 'c#test1': flags.FlagSet(flags.FLAKY), 'c#test2': flags.FlagSet(flags.TIMEOUT), }, suite_runner_util.merge_expectation_map( base_map, {'c#test1': flags.FlagSet(flags.FLAKY)}, flags.FlagSet(flags.TIMEOUT))) # A suite level FLAKY flag should cause all tests to be marked FLAKY, # regardless of whether the base or override expectation is used. self.assertEquals( { 'c#test1': flags.FlagSet(flags.FAIL | flags.LARGE), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map( base_map, {'c#test1': flags.FlagSet(flags.FAIL | flags.LARGE)}, flags.FlagSet(flags.FLAKY))) # A suite level LARGE flag should cause all tests to be marked LARGE, # regardless of whether the base or override expectation is used. self.assertEquals( { 'c#test1': flags.FlagSet(flags.PASS | flags.LARGE), 'c#test2': flags.FlagSet(flags.PASS | flags.LARGE), }, suite_runner_util.merge_expectation_map( base_map, {'c#test2': flags.FlagSet(flags.PASS)}, flags.FlagSet(flags.PASS | flags.LARGE))) with self.assertRaises(AssertionError): # Raise an exception if suite_expectations contains an unknown test name. suite_runner_util.merge_expectation_map( base_map, {'c#test3': flags.FlagSet(flags.PASS)}, flags.FlagSet(flags.PASS))
def test_merge_expectation_map(self): base_map = { 'c#test1': flags.FlagSet(flags.PASS), 'c#test2': flags.FlagSet(flags.FAIL), } # With no override expectations, the base expectations should be used. self.assertEquals( { 'c#test1': flags.FlagSet(flags.PASS), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map( base_map, {}, flags.FlagSet(flags.PASS))) # test1 should be overridden to FAIL, test2 should keep the base FAIL. self.assertEquals( { 'c#test1': flags.FlagSet(flags.FAIL), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map( base_map, {'c#test1': flags.FlagSet(flags.FAIL)}, flags.FlagSet(flags.PASS))) # The pure expectation from the default expectation should end up in the # output expectation map. self.assertEquals( { 'c#test1': flags.FlagSet(flags.FLAKY), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map( base_map, {}, flags.FlagSet(flags.FLAKY))) # If the default expectation is TIMEOUT, all the tests inside should be too # if no other test-level overrides are given self.assertEquals( { 'c#test1': flags.FlagSet(flags.FLAKY), 'c#test2': flags.FlagSet(flags.TIMEOUT), }, suite_runner_util.merge_expectation_map( base_map, {'c#test1': flags.FlagSet(flags.FLAKY)}, flags.FlagSet(flags.TIMEOUT))) # A suite level FLAKY flag should cause all tests to be marked FLAKY, # regardless of whether the base or override expectation is used. self.assertEquals( { 'c#test1': flags.FlagSet(flags.FAIL | flags.LARGE), 'c#test2': flags.FlagSet(flags.FAIL), }, suite_runner_util.merge_expectation_map( base_map, {'c#test1': flags.FlagSet(flags.FAIL | flags.LARGE)}, flags.FlagSet(flags.FLAKY))) # A suite level LARGE flag should cause all tests to be marked LARGE, # regardless of whether the base or override expectation is used. self.assertEquals( { 'c#test1': flags.FlagSet(flags.PASS | flags.LARGE), 'c#test2': flags.FlagSet(flags.PASS | flags.LARGE), }, suite_runner_util.merge_expectation_map( base_map, {'c#test2': flags.FlagSet(flags.PASS)}, flags.FlagSet(flags.PASS | flags.LARGE))) with self.assertRaises(AssertionError): # Raise an exception if suite_expectations contains an unknown test name. suite_runner_util.merge_expectation_map( base_map, {'c#test3': flags.FlagSet(flags.PASS)}, flags.FlagSet(flags.PASS))