def FilterTests(self, test_list, disabled_prefixes=None):
"""Filters |test_list| based on prefixes and, if present, a filter string.
Args:
test_list: The list of tests to filter.
disabled_prefixes: A list of test prefixes to filter. Defaults to
DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_
Returns:
A filtered list of tests to run.
"""
gtest_filter_strings = [
self._GenerateDisabledFilterString(disabled_prefixes)
]
if self._gtest_filter:
gtest_filter_strings.append(self._gtest_filter)
filtered_test_list = test_list
# This lock is required because on older versions of Python
# |unittest_util.FilterTestNames| use of |fnmatch| is not threadsafe.
with self._filter_tests_lock:
for gtest_filter_string in gtest_filter_strings:
logging.debug('Filtering tests using: %s', gtest_filter_string)
filtered_test_list = unittest_util.FilterTestNames(
filtered_test_list, gtest_filter_string)
if self._run_disabled and self._gtest_filter:
out_filtered_test_list = list(
set(test_list) - set(filtered_test_list))
for test in out_filtered_test_list:
test_name_no_disabled = TestNameWithoutDisabledPrefix(test)
if test_name_no_disabled != test and unittest_util.FilterTestNames(
[test_name_no_disabled], self._gtest_filter):
filtered_test_list.append(test)
return filtered_test_list
def gtest_filter(t):
if not filter_str:
return True
# Allow fully-qualified name as well as an omitted package.
unqualified_class_test = {
'class': t['class'].split('.')[-1],
'method': t['method']
}
names = [
GetTestName(t, sep='.'),
GetTestName(unqualified_class_test, sep='.'),
GetUniqueTestName(t, sep='.')
]
if t['is_junit4']:
names += [
GetTestNameWithoutParameterPostfix(t, sep='.'),
GetTestNameWithoutParameterPostfix(unqualified_class_test, sep='.')
]
pattern_groups = filter_str.split('-')
if len(pattern_groups) > 1:
negative_filter = pattern_groups[1]
if unittest_util.FilterTestNames(names, negative_filter):
return []
positive_filter = pattern_groups[0]
return unittest_util.FilterTestNames(names, positive_filter)
def gtest_filter(c, m):
if not test_filter:
return True
# Allow fully-qualified name as well as an omitted package.
names = ['%s.%s' % (c['class'], m['method']),
'%s.%s' % (c['class'].split('.')[-1], m['method'])]
return unittest_util.FilterTestNames(names, test_filter)
def GetAllMatchingTests(self, annotation_filter_list,
exclude_annotation_list, test_filter):
"""Get a list of tests matching any of the annotations and the filter.
Args:
annotation_filter_list: List of test annotations. A test must have at
least one of these annotations. A test without any annotations is
considered to be SmallTest.
exclude_annotation_list: List of test annotations. A test must not have
any of these annotations.
test_filter: Filter used for partial matching on the test method names.
Returns:
List of all matching tests.
"""
if annotation_filter_list:
available_tests = self.GetAnnotatedTests(annotation_filter_list)
# Include un-annotated tests in SmallTest.
if annotation_filter_list.count(self._DEFAULT_ANNOTATION) > 0:
for test in self._GetTestsMissingAnnotation():
logging.warning('%s has no annotations. Assuming "%s".',
test, self._DEFAULT_ANNOTATION)
available_tests.append(test)
else:
available_tests = [
m for m in self.GetTestMethods()
if not self.IsHostDrivenTest(m)
]
if exclude_annotation_list:
excluded_tests = self.GetAnnotatedTests(exclude_annotation_list)
available_tests = list(set(available_tests) - set(excluded_tests))
tests = []
if test_filter:
# |available_tests| are in adb instrument format: package.path.class#test.
# Maps a 'class.test' name to each 'package.path.class#test' name.
sanitized_test_names = dict([(t.split('.')[-1].replace('#',
'.'), t)
for t in available_tests])
# Filters 'class.test' names and populates |tests| with the corresponding
# 'package.path.class#test' names.
tests = [
sanitized_test_names[t] for t in unittest_util.FilterTestNames(
sanitized_test_names.keys(), test_filter.replace('#', '.'))
]
else:
tests = available_tests
# Filter out any tests with SDK level requirements that don't match the set
# of attached devices.
devices = device_utils.DeviceUtils.parallel()
min_sdk_version = min(devices.build_version_sdk.pGet(None))
tests = [
t for t in tests
if self._IsTestValidForSdkRange(t, min_sdk_version)
]
return tests
def Setup(options, devices):
"""Creates a list of test cases and a runner factory.
Returns:
A tuple of (TestRunnerFactory, tests).
"""
test_cases = [
test_case.LinkerLibraryAddressTest,
test_case.LinkerSharedRelroTest,
test_case.LinkerRandomizationTest,
test_case.LinkerLowMemoryThresholdTest ]
low_memory_modes = [False, True]
all_tests = [t(is_low_memory=m) for t in test_cases for m in low_memory_modes]
if options.test_filter:
all_test_names = [ test.qualified_name for test in all_tests ]
filtered_test_names = unittest_util.FilterTestNames(all_test_names,
options.test_filter)
all_tests = [t for t in all_tests \
if t.qualified_name in filtered_test_names]
def TestRunnerFactory(device, shard_index):
return test_runner.LinkerTestRunner(
device, options.tool, options.push_deps,
options.cleanup_test_files)
return (TestRunnerFactory, all_tests)
def Setup(args, _devices):
"""Creates a list of test cases and a runner factory.
Args:
args: an argparse.Namespace object.
Returns:
A tuple of (TestRunnerFactory, tests).
"""
test_cases = [
test_case.LinkerLibraryAddressTest, test_case.LinkerSharedRelroTest,
test_case.LinkerRandomizationTest
]
low_memory_modes = [False, True]
all_tests = [
t(is_low_memory=m) for t in test_cases for m in low_memory_modes
]
if args.test_filter:
all_test_names = [test.qualified_name for test in all_tests]
filtered_test_names = unittest_util.FilterTestNames(
all_test_names, args.test_filter)
all_tests = [t for t in all_tests \
if t.qualified_name in filtered_test_names]
def TestRunnerFactory(device, _shard_index):
return test_runner.LinkerTestRunner(device, args.tool)
return (TestRunnerFactory, all_tests)
def GetTests(self):
tests = [test_case.LinkerSharedRelroTest()]
if self._test_filter:
filtered_names = unittest_util.FilterTestNames(
(t.qualified_name for t in tests), self._test_filter)
tests = [t for t in tests if t.qualified_name in filtered_names]
return tests
def Setup(test_options, devices):
"""Create the test runner factory and tests.
Args:
test_options: A GTestOptions object.
devices: A list of attached devices.
Returns:
A tuple of (TestRunnerFactory, tests).
"""
test_package = test_package_apk.TestPackageApk(test_options.suite_name)
if not os.path.exists(test_package.suite_path):
exe_test_package = test_package_exe.TestPackageExecutable(
test_options.suite_name)
if not os.path.exists(exe_test_package.suite_path):
raise Exception(
'Did not find %s target. Ensure it has been built.\n'
'(not found at %s or %s)'
% (test_options.suite_name,
test_package.suite_path,
exe_test_package.suite_path))
test_package = exe_test_package
logging.warning('Found target %s', test_package.suite_path)
_GenerateDepsDirUsingIsolate(test_options.suite_name,
test_options.isolate_file_path)
tests = _GetTests(test_options, test_package, devices)
# Constructs a new TestRunner with the current options.
def TestRunnerFactory(device, _shard_index):
return test_runner.TestRunner(
test_options,
device,
test_package)
if test_options.run_disabled:
test_options = test_options._replace(
test_arguments=('%s --gtest_also_run_disabled_tests' %
test_options.test_arguments))
else:
tests = _FilterDisabledTests(tests, test_options.suite_name,
bool(test_options.gtest_filter))
if test_options.gtest_filter:
tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
# Coalesce unit tests into a single test per device
if test_options.suite_name != 'content_browsertests':
num_devices = len(devices)
tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
tests = [t for t in tests if t]
return (TestRunnerFactory, tests)
def gtest_filter(t):
if not test_filter:
return True
# Allow fully-qualified name as well as an omitted package.
unqualified_class_test = {
'class': t['class'].split('.')[-1],
'method': t['method']
}
names = [
GetTestName(t, sep='.'),
GetTestName(unqualified_class_test, sep='.'),
GetUniqueTestName(t, sep='.')
]
return unittest_util.FilterTestNames(names, test_filter)
def GetAvailableTests(self, annotations, exclude_annotations, name_filter):
"""Get a collection of TestInfos which match the supplied criteria.
Args:
annotations: List of annotations. Each test in the returned list is
annotated with atleast one of these annotations.
exclude_annotations: List of annotations. The tests in the returned
list are not annotated with any of these annotations.
name_filter: name filter which tests must match, if any
Returns:
List of available tests.
"""
available_tests = self.all_tests
# Filter out tests which match neither the requested annotation, nor the
# requested name filter, if any.
available_tests = [
t for t in available_tests
if self._AnnotationIncludesTest(t, annotations)
]
if annotations and len(
annotations) == 1 and annotations[0] == 'SmallTest':
tests_without_annotation = [
t for t in self.all_tests
if not tests_annotations.AnnotatedFunctions.GetTestAnnotations(
t.qualified_name)
]
test_names = [t.qualified_name for t in tests_without_annotation]
logging.warning(
'The following tests do not contain any annotation. '
'Assuming "SmallTest":\n%s', '\n'.join(test_names))
available_tests += tests_without_annotation
if exclude_annotations:
excluded_tests = [
t for t in available_tests
if self._AnnotationIncludesTest(t, exclude_annotations)
]
available_tests = list(set(available_tests) - set(excluded_tests))
if name_filter:
available_test_names = unittest_util.FilterTestNames(
[t.qualified_name for t in available_tests], name_filter)
available_tests = [
t for t in available_tests
if t.qualified_name in available_test_names
]
return available_tests
def GetTests(self, min_device_sdk):
tests = [
test_case.LinkerSharedRelroTest(is_modern_linker=False,
is_low_memory=False),
test_case.LinkerSharedRelroTest(is_modern_linker=False,
is_low_memory=True)
]
if min_device_sdk >= _MODERN_LINKER_MINIMUM_SDK_INT:
tests.append(
test_case.LinkerSharedRelroTest(is_modern_linker=True))
if self._test_filter:
filtered_names = unittest_util.FilterTestNames(
(t.qualified_name for t in tests), self._test_filter)
tests = [t for t in tests if t.qualified_name in filtered_names]
return tests
def Setup(args, devices):
"""Creates a list of test cases and a runner factory.
Args:
args: an argparse.Namespace object.
devices: an iterable of available devices.
Returns:
A tuple of (TestRunnerFactory, tests).
"""
legacy_linker_tests = [
test_case.LinkerSharedRelroTest(is_modern_linker=False,
is_low_memory=False),
test_case.LinkerSharedRelroTest(is_modern_linker=False,
is_low_memory=True),
]
modern_linker_tests = [
test_case.LinkerSharedRelroTest(is_modern_linker=True),
]
min_sdk_int = 1 << 31
for device in devices:
min_sdk_int = min(min_sdk_int, device.build_version_sdk)
if min_sdk_int >= _MODERN_LINKER_MINIMUM_SDK_INT:
all_tests = legacy_linker_tests + modern_linker_tests
else:
all_tests = legacy_linker_tests
logging.warn(
'Not running LinkerModern tests (requires API %d, found %d)',
_MODERN_LINKER_MINIMUM_SDK_INT, min_sdk_int)
if args.test_filter:
all_test_names = [test.qualified_name for test in all_tests]
filtered_test_names = unittest_util.FilterTestNames(
all_test_names, args.test_filter)
all_tests = [t for t in all_tests \
if t.qualified_name in filtered_test_names]
def TestRunnerFactory(device, _shard_index):
return test_runner.LinkerTestRunner(device, args.tool)
return (TestRunnerFactory, all_tests)
def test_names_from_pattern(combined_pattern, test_names):
patterns = combined_pattern.split(':')
hashable_patterns = set()
filename_patterns = []
for pattern in patterns:
if ('*' in pattern or '?' in pattern or '[' in pattern):
filename_patterns.append(pattern)
else:
hashable_patterns.add(pattern)
filter_test_names = set(
unittest_util.FilterTestNames(test_names, ':'.join(
filename_patterns))) if len(filename_patterns) > 0 else set()
for test_name in test_names:
if test_name in hashable_patterns:
filter_test_names.add(test_name)
return filter_test_names
def _FilterDisabledTests(tests, suite_name, has_gtest_filter):
"""Removes disabled tests from |tests|.
Applies the following filters in order:
1. Remove tests with disabled prefixes.
2. Remove tests specified in the *_disabled files in the 'filter' dir
Args:
tests: List of tests.
suite_name: Name of the test suite (e.g. base_unittests).
has_gtest_filter: Whether a gtest_filter is provided.
Returns:
List of tests remaining.
"""
tests = _FilterTestsUsingPrefixes(
tests, has_gtest_filter, has_gtest_filter)
tests = unittest_util.FilterTestNames(
tests, _GetDisabledTestsFilterFromFile(suite_name))
return tests
def FilterTests(self, test_list, disabled_prefixes=None):
"""Filters |test_list| based on prefixes and, if present, a filter string.
Args:
test_list: The list of tests to filter.
disabled_prefixes: A list of test prefixes to filter. Defaults to
DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_
Returns:
A filtered list of tests to run.
"""
gtest_filter_strings = [
self._GenerateDisabledFilterString(disabled_prefixes)]
if self._gtest_filter:
gtest_filter_strings.append(self._gtest_filter)
filtered_test_list = test_list
for gtest_filter_string in gtest_filter_strings:
logging.debug('Filtering tests using: %s', gtest_filter_string)
filtered_test_list = unittest_util.FilterTestNames(
filtered_test_list, gtest_filter_string)
return filtered_test_list
def testMatchAll(self):
x = unittest_util.FilterTestNames(self.possible_list, "*")
self.assertEquals(x, self.possible_list)
def testMatchFull(self):
x = unittest_util.FilterTestNames(self.possible_list, "Foo.Two")
self.assertEquals(x, ["Foo.Two"])
def testMatchWithNegative(self):
x = unittest_util.FilterTestNames(self.possible_list,
"Bar.*:Foo.*-*.Three")
self.assertEquals(x, ["Bar.One", "Bar.Two", "Foo.One", "Foo.Two"])
def testMatchOverlapping(self):
x = unittest_util.FilterTestNames(self.possible_list, "Bar.*:*.Two")
self.assertEquals(
x, ["Bar.One", "Bar.Two", "Bar.Three", "Foo.Two", "Quux.Two"])
def Setup(test_options, devices):
"""Create the test runner factory and tests.
Args:
test_options: A GTestOptions object.
devices: A list of attached devices.
Returns:
A tuple of (TestRunnerFactory, tests).
"""
test_package = test_package_apk.TestPackageApk(test_options.suite_name)
if not os.path.exists(test_package.suite_path):
exe_test_package = test_package_exe.TestPackageExecutable(
test_options.suite_name)
if not os.path.exists(exe_test_package.suite_path):
raise Exception(
'Did not find %s target. Ensure it has been built.\n'
'(not found at %s or %s)'
% (test_options.suite_name,
test_package.suite_path,
exe_test_package.suite_path))
test_package = exe_test_package
logging.warning('Found target %s', test_package.suite_path)
base_setup.GenerateDepsDirUsingIsolate(test_options.suite_name,
test_options.isolate_file_path,
ISOLATE_FILE_PATHS,
DEPS_EXCLUSION_LIST)
def push_data_deps_to_device_dir(device):
device_dir = (constants.TEST_EXECUTABLE_DIR
if test_package.suite_name == 'breakpad_unittests'
else device.GetExternalStoragePath())
base_setup.PushDataDeps(device, device_dir, test_options)
device_utils.DeviceUtils.parallel(devices).pMap(push_data_deps_to_device_dir)
tests = _GetTests(test_options, test_package, devices)
# Constructs a new TestRunner with the current options.
def TestRunnerFactory(device, _shard_index):
return test_runner.TestRunner(
test_options,
device,
test_package)
if test_options.run_disabled:
test_options = test_options._replace(
test_arguments=('%s --gtest_also_run_disabled_tests' %
test_options.test_arguments))
else:
tests = _FilterDisabledTests(tests, test_options.suite_name,
bool(test_options.gtest_filter))
if test_options.gtest_filter:
tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
# Coalesce unit tests into a single test per device
if test_options.suite_name != 'content_browsertests':
num_devices = len(devices)
tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
tests = [t for t in tests if t]
return (TestRunnerFactory, tests)
def gtest_filter(c, m):
t = ['%s.%s' % (c['class'].split('.')[-1], m['method'])]
return (not self._test_filter
or unittest_util.FilterTestNames(t, self._test_filter))
def testMatchPartial(self):
x = unittest_util.FilterTestNames(self.possible_list, "Foo.*")
self.assertEquals(x, ["Foo.One", "Foo.Two", "Foo.Three"])