def main(host=None, args=None):
options = parse_args(args)
if options.no_autopep8:
options.style = None
if options.leave_strings_alone:
options.quoting = None
autopep8_options = _autopep8_options_for_style(options.style)
fixers = ['webkitpy.formatter.fix_docstrings']
fixers.extend(_fixers_for_quoting(options.quoting))
if options.files == ['-']:
host = host or SystemHost()
host.print_(reformat_source(host.stdin.read(), autopep8_options,
fixers, '<stdin>'),
end='')
return
# We create the arglist before checking if we need to create a Host, because a
# real host is non-picklable and can't be passed to host.executive.map().
arglist = [(host, name, autopep8_options, fixers, options.backup)
for name in options.files]
host = host or SystemHost()
host.executive.map(_reformat_thunk, arglist, processes=options.jobs)
def setUp(self):
# A real PlatformInfo object is used here instead of a
# MockPlatformInfo because we need to actually check for
# Windows and Mac to skip some tests.
self._platform = SystemHost().platform
# FIXME: Remove this when we fix test-webkitpy to work
# properly on cygwin (bug 63846).
self.should_test_processes = not self._platform.is_win()
def __init__(self):
SystemHost.__init__(self)
self._checkout = None
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self._engage_awesome_locale_hacks()
def _create_or_find_device_for_request(request, host=SystemHost(), name_base='Managed'):
assert isinstance(request, DeviceRequest)
device = SimulatedDeviceManager._find_exisiting_device_for_request(request)
if device:
return device
name = SimulatedDeviceManager._find_available_name(name_base)
device_type = SimulatedDeviceManager._disambiguate_device_type(request.device_type)
runtime = SimulatedDeviceManager.get_runtime_for_device_type(device_type)
device_identifier = SimulatedDeviceManager._get_device_identifier_for_type(device_type)
assert runtime is not None
assert device_identifier is not None
for device in SimulatedDeviceManager.available_devices(host):
if device.platform_device.name == name:
device.platform_device._delete()
break
_log.debug(u"Creating device '{}', of type {}".format(name, device_type))
host.executive.run_command([SimulatedDeviceManager.xcrun, 'simctl', 'create', name, device_identifier, runtime.identifier])
# We just added a device, so our list of _available_devices needs to be re-synced.
SimulatedDeviceManager.populate_available_devices(host)
for device in SimulatedDeviceManager.available_devices(host):
if device.platform_device.name == name:
device.platform_device.managed_by_script = True
return device
return None
def test_virtual_test_suites(self):
# We test that we can load the real LayoutTests/VirtualTestSuites file properly, so we
# use a real SystemHost(). We don't care what virtual_test_suites() returns as long
# as it is iterable.
port = self.make_port(host=SystemHost(), port_name=self.full_port_name)
self.assertTrue(
isinstance(port.virtual_test_suites(), collections.Iterable))
def populate_available_devices(host=SystemHost()):
if not host.platform.is_mac():
return
try:
simctl_json = json.loads(
host.executive.run_command(
[SimulatedDeviceManager.xcrun, 'simctl', 'list',
'--json']))
except (ValueError, ScriptError):
return
SimulatedDeviceManager._device_identifier_to_name = {
device['identifier']: device['name']
for device in simctl_json['devicetypes']
}
SimulatedDeviceManager.AVAILABLE_RUNTIMES = SimulatedDeviceManager._create_runtimes(
simctl_json['runtimes'])
for runtime in SimulatedDeviceManager.AVAILABLE_RUNTIMES:
for device_json in simctl_json['devices'][runtime.name]:
device = SimulatedDeviceManager._create_device_with_runtime(
host, runtime, device_json)
if not device:
continue
# Update device state from simctl output.
device.platform_device._state = SimulatedDevice.NAME_FOR_STATE.index(
device_json['state'].upper())
device.platform_device._last_updated_state = time.time()
return
def test_basic(self):
cmd = [sys.executable, '-c', 'import sys; import time; time.sleep(0.02); print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"']
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
self.assertEqual(proc.poll(), None)
self.assertFalse(proc.has_crashed())
# check that doing a read after an expired deadline returns
# nothing immediately.
line = proc.read_stdout_line(now - 1)
self.assertEqual(line, None)
# FIXME: This part appears to be flaky. line should always be non-None.
# FIXME: https://bugs.webkit.org/show_bug.cgi?id=88280
line = proc.read_stdout_line(now + 1.0)
if line:
self.assertEqual(line.strip(), "stdout")
line = proc.read_stderr_line(now + 1.0)
if line:
self.assertEqual(line.strip(), "stderr")
proc.stop(0)
def swap(device,
request,
host=SystemHost(),
name_base='Managed',
timeout=SIMULATOR_BOOT_TIMEOUT):
if SimulatedDeviceManager.INITIALIZED_DEVICES is None:
raise RuntimeError(
'Cannot swap when there are no initialized devices')
if device not in SimulatedDeviceManager.INITIALIZED_DEVICES:
raise RuntimeError(
'{} is not initialized, cannot swap it'.format(device))
index = SimulatedDeviceManager.INITIALIZED_DEVICES.index(device)
SimulatedDeviceManager.INITIALIZED_DEVICES[index] = None
device.platform_device._tear_down()
device = SimulatedDeviceManager._create_or_find_device_for_request(
request, host, name_base)
assert device
if not device.platform_device.is_booted_or_booting(force_update=True):
device.platform_device.booted_by_script = True
_log.debug("Booting device '{}'".format(device.udid))
host.executive.run_command(
[SimulatedDeviceManager.xcrun, 'simctl', 'boot', device.udid])
SimulatedDeviceManager.INITIALIZED_DEVICES[index] = device
deadline = time.time() + timeout
SimulatedDeviceManager._wait_until_device_is_usable(
device, max(0, deadline - time.time()))
def populate_available_devices(host=SystemHost()):
if not host.platform.is_mac():
return
try:
simctl_json = json.loads(host.executive.run_command([SimulatedDeviceManager.xcrun, 'simctl', 'list', '--json'], decode_output=False))
except (ValueError, ScriptError):
return
SimulatedDeviceManager._device_identifier_to_name = {device['identifier']: device['name'] for device in simctl_json['devicetypes']}
SimulatedDeviceManager.AVAILABLE_RUNTIMES = SimulatedDeviceManager._create_runtimes(simctl_json['runtimes'])
for runtime in SimulatedDeviceManager.AVAILABLE_RUNTIMES:
# Needed for <rdar://problem/47122965>
devices = []
if isinstance(simctl_json['devices'], list):
for devices_for_runtime in simctl_json['devices']:
if devices_for_runtime['name'] == runtime.name:
devices = devices_for_runtime['devices']
break
else:
devices = simctl_json['devices'].get(runtime.name, None) or simctl_json['devices'].get(runtime.identifier, [])
for device_json in devices:
device = SimulatedDeviceManager._create_device_with_runtime(host, runtime, device_json)
if not device:
continue
# Update device state from simctl output.
device.platform_device._state = SimulatedDevice.NAME_FOR_STATE.index(device_json['state'].upper())
device.platform_device._last_updated_state = time.time()
return
def setup(cls):
LLDB_WEBKIT_TESTER_NAME = 'lldbWebKitTester'
BREAK_FOR_TESTING_FUNCTION_NAME = 'breakForTestingSummaryProviders'
cls.sbDebugger = lldb.SBDebugger.Create()
cls.sbDebugger.SetAsync(False)
host = SystemHost()
config = Config(host.executive, host.filesystem)
cls.lldbWebKitTesterExecutable = os.path.join(config.build_directory(config.default_configuration()), LLDB_WEBKIT_TESTER_NAME)
cls.sbTarget = cls.sbDebugger.CreateTarget(str(cls.lldbWebKitTesterExecutable))
assert cls.sbTarget
cls.sbTarget.BreakpointCreateByName(BREAK_FOR_TESTING_FUNCTION_NAME, cls.sbTarget.GetExecutable().GetFilename())
argv = None
envp = None
cls.sbProcess = cls.sbTarget.LaunchSimple(argv, envp, os.getcwd())
assert cls.sbProcess
assert cls.sbProcess.GetState() == lldb.eStateStopped
cls.sbThread = cls.sbProcess.GetThreadAtIndex(0)
assert cls.sbThread
# Frame 0 is the function with name BREAK_FOR_TESTING_FUNCTION_NAME. We want the frame of the caller of
# BREAK_FOR_TESTING_FUNCTION_NAME because it has all the interesting local variables we want to test.
cls.sbFrame = cls.sbThread.GetFrameAtIndex(1)
assert cls.sbFrame
def __init__(self):
SystemHost.__init__(self)
self.web = web.Web()
self._scm = None
# Everything below this line is WebKit-specific and belongs on a higher-level object.
self.buildbot = buildbot.BuildBot()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self._engage_awesome_locale_hacks()
def create_crash_logs_darwin(self):
if not SystemHost().platform.is_mac():
return
self.older_mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28528)
self.mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28530)
self.newer_mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28529)
self.other_process_mock_crash_report = make_mock_crash_report_darwin(
'FooProcess', 28527)
self.misformatted_mock_crash_report = 'Junk that should not appear in a crash report' + make_mock_crash_report_darwin(
'DumpRenderTree', 28526)[200:]
self.files = {}
self.files[
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150718_quadzen.crash'] = self.older_mock_crash_report
self.files[
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash'] = self.mock_crash_report
self.files[
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150720_quadzen.crash'] = self.newer_mock_crash_report
self.files[
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150721_quadzen.crash'] = None
self.files[
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150722_quadzen.crash'] = self.other_process_mock_crash_report
self.files[
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150723_quadzen.crash'] = self.misformatted_mock_crash_report
self.filesystem = MockFileSystem(self.files)
crash_logs = CrashLogs(MockSystemHost(filesystem=self.filesystem),
CrashLogsTest.DARWIN_MOCK_CRASH_DIRECTORY)
logs = self.filesystem.files_under(
'/Users/mock/Library/Logs/DiagnosticReports/')
for path in reversed(sorted(logs)):
self.assertTrue(path in self.files.keys())
return crash_logs
def serial_test_process_crashing(self):
# Give -u switch to force stdout to be unbuffered for Windows
cmd = [
sys.executable, '-uc',
'import sys; print("stdout 1"); print("stdout 2"); print("stdout 3"); sys.stdin.readline(); sys.exit(1);'
]
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write(b'')
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line.strip(), b'stdout 1')
proc.write(b'End\n')
time.sleep(0.1) # Give process a moment to close.
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line.strip(), b'stdout 2')
self.assertEqual(True, proc.has_crashed())
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line, None)
proc.stop(0)
def test_find_log_darwin(self):
if not SystemHost().platform.is_mac():
return
crash_logs = self.create_crash_logs_darwin()
log = crash_logs.find_newest_log("DumpRenderTree")
self.assertMultiLineEqual(log, self.newer_mock_crash_report)
log = crash_logs.find_newest_log("DumpRenderTree", 28529)
self.assertMultiLineEqual(log, self.newer_mock_crash_report)
log = crash_logs.find_newest_log("DumpRenderTree", 28530)
self.assertMultiLineEqual(log, self.mock_crash_report)
log = crash_logs.find_newest_log("DumpRenderTree", 28531)
self.assertIsNone(log)
log = crash_logs.find_newest_log("DumpRenderTree", newer_than=1.0)
self.assertIsNone(log)
def bad_read(path):
raise IOError('IOError: No such file or directory')
def bad_mtime(path):
raise OSError('OSError: No such file or directory')
self.filesystem.read_text_file = bad_read
log = crash_logs.find_newest_log("DumpRenderTree", 28531, include_errors=True)
self.assertIn('IOError: No such file or directory', log)
self.filesystem = MockFileSystem(self.files)
crash_logs = CrashLogs(MockSystemHost(filesystem=self.filesystem))
self.filesystem.mtime = bad_mtime
log = crash_logs.find_newest_log("DumpRenderTree", newer_than=1.0, include_errors=True)
self.assertIn('OSError: No such file or directory', log)
def test_basic(self):
cmd = [
sys.executable, '-c',
'import sys; print "stdout"; sys.stdout.flush(); print >>sys.stderr, "stderr"; sys.stdin.readline();'
]
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
if sys.platform.startswith('win'):
self.assertEqual(proc.poll(), 0)
else:
self.assertEqual(proc.poll(), None)
self.assertFalse(proc.has_crashed())
# check that doing a read after an expired deadline returns
# nothing immediately.
line = proc.read_stdout_line(now - 1)
self.assertEqual(line, None)
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line.strip(), "stdout")
line = proc.read_stderr_line(now + 1.0)
self.assertEqual(line.strip(), "stderr")
proc.write('End\n')
proc.stop(0)
def _boot_device(device, host=None):
host = host or SystemHost()
_log.debug(u"Booting device '{}'".format(device.udid))
device.platform_device.booted_by_script = True
host.executive.run_command(
[SimulatedDeviceManager.xcrun, 'simctl', 'boot', device.udid])
SimulatedDeviceManager.INITIALIZED_DEVICES.append(device)
def test_process_crashing(self):
cmd = [
sys.executable, '-c',
'import sys; print "stdout 1"; print "stdout 2"; print "stdout 3"; sys.stdout.flush(); sys.stdin.readline(); sys.exit(1);'
]
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line.strip(), 'stdout 1')
proc.write('End\n')
time.sleep(0.1) # Give process a moment to close.
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line.strip(), 'stdout 2')
self.assertEqual(True, proc.has_crashed())
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line, None)
proc.stop(0)
def __init__(self):
SystemHost.__init__(self)
self.web = web.Web()
self._scm = None
# Everything below this line is WebKit-specific and belongs on a higher-level object.
self.buildbot = buildbot.BuildBot()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, scm, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self._engage_awesome_locale_hacks()
def max_supported_simulators(host=SystemHost()):
if not host.platform.is_mac():
return 0
try:
system_process_count_limit = int(host.executive.run_command(['/usr/bin/ulimit', '-u']).strip())
current_process_count = len(host.executive.run_command(['/bin/ps', 'aux']).strip().split('\n'))
_log.debug(u'Process limit: {}, current #processes: {}'.format(system_process_count_limit, current_process_count))
except (ValueError, ScriptError):
return 0
max_supported_simulators_for_hardware = min(
host.executive.cpu_count() // 2,
host.platform.total_bytes_memory() // SimulatedDeviceManager.MEMORY_ESTIMATE_PER_SIMULATOR_INSTANCE,
SimulatedDeviceManager.MAX_NUMBER_OF_SIMULATORS,
)
max_supported_simulators_locally = (system_process_count_limit - current_process_count) // SimulatedDeviceManager.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
if (max_supported_simulators_locally < max_supported_simulators_for_hardware):
_log.warn(u'This machine could support {} simulators, but is only configured for {}.'.format(max_supported_simulators_for_hardware, max_supported_simulators_locally))
_log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
if max_supported_simulators_locally == 0:
max_supported_simulators_locally = 1
return min(max_supported_simulators_locally, max_supported_simulators_for_hardware)
def tear_down(host=None, timeout=SIMULATOR_BOOT_TIMEOUT):
host = host or SystemHost()
if SimulatedDeviceManager._managing_simulator_app:
host.executive.run_command(['killall', '-9', 'Simulator'], return_exit_code=True)
SimulatedDeviceManager._managing_simulator_app = False
if SimulatedDeviceManager.INITIALIZED_DEVICES is None:
return
deadline = time.time() + timeout
while SimulatedDeviceManager.INITIALIZED_DEVICES:
device = SimulatedDeviceManager.INITIALIZED_DEVICES[0]
if device is None:
SimulatedDeviceManager.INITIALIZED_DEVICES.remove(None)
continue
device.platform_device._tear_down(deadline - time.time())
SimulatedDeviceManager.INITIALIZED_DEVICES = None
if SimulatedDeviceManager._managing_simulator_app:
for pid in host.executive.running_pids(lambda name: 'CoreSimulator.framework' in name):
host.executive.kill_process(pid)
# If we were managing the simulator, there are some cache files we need to remove
for directory in host.filesystem.glob('/tmp/com.apple.CoreSimulator.SimDevice.*'):
host.filesystem.rmtree(directory)
core_simulator_directory = host.filesystem.expanduser(host.filesystem.join('~', 'Library', 'Developer', 'CoreSimulator'))
host.filesystem.rmtree(host.filesystem.join(core_simulator_directory, 'Caches'))
host.filesystem.rmtree(host.filesystem.join(core_simulator_directory, 'Temp'))
def serial_test_basic(self):
# Give -u switch to force stdout and stderr to be unbuffered for Windows
cmd = [
sys.executable, '-uc',
'import sys; print "stdout"; print >>sys.stderr, "stderr"; sys.stdin.readline();'
]
host = SystemHost()
factory = PortFactory(host)
port = factory.get()
now = time.time()
proc = server_process.ServerProcess(port, 'python', cmd)
proc.write('')
self.assertEqual(proc.poll(), None)
self.assertFalse(proc.has_crashed())
# check that doing a read after an expired deadline returns
# nothing immediately.
line = proc.read_stdout_line(now - 1)
self.assertEqual(line, None)
line = proc.read_stdout_line(now + 1.0)
self.assertEqual(line.strip(), "stdout")
line = proc.read_stderr_line(now + 1.0)
self.assertEqual(line.strip(), "stderr")
proc.write('End\n')
time.sleep(0.1) # Give process a moment to close.
self.assertEqual(proc.poll(), 0)
proc.stop(0)
def test_find_log_darwin(self):
if not SystemHost().platform.is_mac():
return
older_mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28528)
mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28530)
newer_mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28529)
other_process_mock_crash_report = make_mock_crash_report_darwin(
'FooProcess', 28527)
misformatted_mock_crash_report = 'Junk that should not appear in a crash report' + \
make_mock_crash_report_darwin('DumpRenderTree', 28526)[200:]
files = {
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150718_quadzen.crash':
older_mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash':
mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150720_quadzen.crash':
newer_mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150721_quadzen.crash':
None,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150722_quadzen.crash':
other_process_mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150723_quadzen.crash':
misformatted_mock_crash_report,
}
filesystem = MockFileSystem(files)
crash_logs = CrashLogs(MockSystemHost(filesystem=filesystem))
log = crash_logs.find_newest_log("DumpRenderTree")
self.assertMultiLineEqual(log, newer_mock_crash_report)
log = crash_logs.find_newest_log("DumpRenderTree", 28529)
self.assertMultiLineEqual(log, newer_mock_crash_report)
log = crash_logs.find_newest_log("DumpRenderTree", 28530)
self.assertMultiLineEqual(log, mock_crash_report)
log = crash_logs.find_newest_log("DumpRenderTree", 28531)
self.assertIsNone(log)
log = crash_logs.find_newest_log("DumpRenderTree", newer_than=1.0)
self.assertIsNone(log)
def bad_read(path):
raise IOError('IOError: No such file or directory')
def bad_mtime(path):
raise OSError('OSError: No such file or directory')
filesystem.read_text_file = bad_read
log = crash_logs.find_newest_log("DumpRenderTree",
28531,
include_errors=True)
self.assertIn('IOError: No such file or directory', log)
filesystem = MockFileSystem(files)
crash_logs = CrashLogs(MockSystemHost(filesystem=filesystem))
filesystem.mtime = bad_mtime
log = crash_logs.find_newest_log("DumpRenderTree",
newer_than=1.0,
include_errors=True)
self.assertIn('OSError: No such file or directory', log)
def device_by_filter(filter, host=None):
host = host or SystemHost()
result = []
for device in SimulatedDeviceManager.available_devices(host):
if filter(device):
result.append(device)
return result
def configure(self, options):
self.options = options
if options.timing:
# --timing implies --verbose
options.verbose = max(options.verbose, 1)
log_level = logging.INFO
if options.quiet:
log_level = logging.WARNING
elif options.verbose == 2:
log_level = logging.DEBUG
self.meter = MeteredStream(
self.stream, (options.verbose == 2),
number_of_columns=SystemHost().platform.terminal_width())
handler = logging.StreamHandler(self.stream)
# We constrain the level on the handler rather than on the root
# logger itself. This is probably better because the handler is
# configured and known only to this module, whereas the root logger
# is an object shared (and potentially modified) by many modules.
# Modifying the handler, then, is less intrusive and less likely to
# interfere with modifications made by other modules (e.g. in unit
# tests).
handler.name = __name__
handler.setLevel(log_level)
formatter = logging.Formatter("%(message)s")
handler.setFormatter(formatter)
logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(logging.NOTSET)
# Filter out most webkitpy messages.
#
# Messages can be selectively re-enabled for this script by updating
# this method accordingly.
def filter_records(record):
"""Filter out autoinstall and non-third-party webkitpy messages."""
# FIXME: Figure out a way not to use strings here, for example by
# using syntax like webkitpy.test.__name__. We want to be
# sure not to import any non-Python 2.4 code, though, until
# after the version-checking code has executed.
if (record.name.startswith("webkitpy.common.system.autoinstall")
or record.name.startswith("webkitpy.test")):
return True
if record.name.startswith("webkitpy"):
return False
return True
testing_filter = logging.Filter()
testing_filter.filter = filter_records
# Display a message so developers are not mystified as to why
# logging does not work in the unit tests.
_log.info(
"Suppressing most webkitpy logging while running unit tests.")
handler.addFilter(testing_filter)
def main(host=None, args=None):
options = parse_args(args)
if options.files == ['-']:
host = host or SystemHost()
host.print_(reformat_source(host.stdin.read(), options.style,
'<stdin>'),
end='')
return
# We create the arglist before checking if we need to create a Host, because a
# real host is non-picklable and can't be passed to host.executive.map().
arglist = [(host, name, options.style, options.backup)
for name in options.files]
host = host or SystemHost()
host.executive.map(_reformat_thunk, arglist, processes=options.jobs)
def __init__(self, file_path, handle_style_error, scm=None, host=None):
self._file_path = file_path
self._handle_style_error = handle_style_error
self._host = host or SystemHost()
self._fs = self._host.filesystem
self._detector = scm or SCMDetector(
self._fs, self._host.executive).detect_scm_system(
self._fs.getcwd())
def reformat_file(host, name, autopep8_options, fixers, should_backup_file):
host = host or SystemHost()
source = host.filesystem.read_text_file(name)
dest = reformat_source(source, autopep8_options, fixers, name)
if dest != source:
if should_backup_file:
host.filesystem.write_text_file(name + '.bak', source)
host.filesystem.write_text_file(name, dest)
def __init__(self, tool, options, host=None, scm=None):
self._tool = tool
self._options = options
self._host = host or SystemHost()
self._fs = self._host.filesystem
self._detector = scm or SCMDetector(
self._fs, self._host.executive).detect_scm_system(
self._fs.getcwd())
def _boot_device(device, host=None):
host = host or SystemHost()
_log.debug(u"Booting device '{}'".format(device.udid))
device.platform_device.booted_by_script = True
host.executive.run_command([SimulatedDeviceManager.xcrun, 'simctl', 'boot', device.udid])
SimulatedDeviceManager.INITIALIZED_DEVICES.append(device)
# FIXME: Remove this delay once rdar://77234240 is resolved.
time.sleep(10)
def setUp(self):
# A real PlatformInfo object is used here instead of a
# MockPlatformInfo because we need to actually check for
# Windows and Mac to skip some tests.
self._platform = SystemHost().platform
# FIXME: Remove this when we fix test-webkitpy to work
# properly on cygwin (bug 63846).
self.should_test_processes = not self._platform.is_win()
def test_find_all_log_darwin(self):
if not SystemHost().platform.is_mac():
return
crash_logs = self.create_crash_logs_darwin()
all_logs = crash_logs.find_all_logs()
self.assertEqual(len(all_logs), 5)
for test, crash_log in all_logs.iteritems():
self.assertTrue(crash_log in self.files.values())
self.assertTrue(test == "Unknown" or int(test.split("-")[1]) in range(28527, 28531))
def initialize_devices(requests, host=None, name_base='Managed', simulator_ui=True, timeout=SIMULATOR_BOOT_TIMEOUT, **kwargs):
host = host or SystemHost()
if SimulatedDeviceManager.INITIALIZED_DEVICES is not None:
return SimulatedDeviceManager.INITIALIZED_DEVICES
if not host.platform.is_mac():
return None
SimulatedDeviceManager.INITIALIZED_DEVICES = []
atexit.register(SimulatedDeviceManager.tear_down)
# Convert to iterable type
if not hasattr(requests, '__iter__'):
requests = [requests]
# Check running sims
for device in SimulatedDeviceManager.available_devices(host):
matched_request = SimulatedDeviceManager._does_fulfill_request(device, requests)
if matched_request is None:
continue
requests.remove(matched_request)
_log.debug(u'Attached to running simulator {}'.format(device))
SimulatedDeviceManager.INITIALIZED_DEVICES.append(device)
# DeviceRequests are compared by reference
requests_copy = [request for request in requests]
# Merging requests means that if 4 devices are requested, but only one is running, these
# 4 requests will be fulfilled by the 1 running device.
for request in requests_copy:
if not request.merge_requests:
continue
if not request.use_booted_simulator:
continue
if request.device_type != device.device_type and not request.allow_incomplete_match:
continue
if request.device_type.software_variant != device.device_type.software_variant:
continue
requests.remove(request)
for request in requests:
device = SimulatedDeviceManager._create_or_find_device_for_request(request, host, name_base)
assert device is not None
SimulatedDeviceManager._boot_device(device, host)
if simulator_ui and host.executive.run_command(['killall', '-0', 'Simulator'], return_exit_code=True) != 0:
SimulatedDeviceManager._managing_simulator_app = not host.executive.run_command(['open', '-g', '-b', SimulatedDeviceManager.simulator_bundle_id, '--args', '-PasteboardAutomaticSync', '0'], return_exit_code=True)
deadline = time.time() + timeout
for device in SimulatedDeviceManager.INITIALIZED_DEVICES:
SimulatedDeviceManager._wait_until_device_is_usable(device, deadline)
return SimulatedDeviceManager.INITIALIZED_DEVICES
class RunTest(unittest.TestCase, StreamTestingMixin):
def setUp(self):
# A real PlatformInfo object is used here instead of a
# MockPlatformInfo because we need to actually check for
# Windows and Mac to skip some tests.
self._platform = SystemHost().platform
# FIXME: Remove this when we fix test-webkitpy to work
# properly on cygwin (bug 63846).
self.should_test_processes = not self._platform.is_win()
def test_basic(self):
options, args = parse_args(tests_included=True)
logging_stream = StringIO.StringIO()
host = MockHost()
port_obj = host.port_factory.get(options.platform, options)
details = run_webkit_tests.run(port_obj, options, args, logging_stream)
# These numbers will need to be updated whenever we add new tests.
self.assertEqual(details.initial_results.total, test.TOTAL_TESTS)
self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS)
self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES)
self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES)
self.assertEqual(details.retry_results.total, test.TOTAL_RETRIES)
one_line_summary = "%d tests ran as expected, %d didn't:\n" % (
details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name),
len(details.initial_results.unexpected_results_by_name))
self.assertTrue(one_line_summary in logging_stream.buflist)
# Ensure the results were summarized properly.
self.assertEqual(details.summarized_results['num_regressions'], details.exit_code)
# Ensure the image diff percentage is in the results.
self.assertEqual(details.summarized_results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1)
# Ensure the results were written out and displayed.
full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
self.assertEqual(json.loads(json_to_eval), details.summarized_results)
self.assertEqual(host.user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def test_batch_size(self):
batch_tests_run = get_test_batches(['--batch-size', '2'])
for batch in batch_tests_run:
self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
def test_max_locked_shards(self):
# Tests for the default of using one locked shard even in the case of more than one child process.
if not self.should_test_processes:
return
save_env_webkit_test_max_locked_shards = None
if "WEBKIT_TEST_MAX_LOCKED_SHARDS" in os.environ:
save_env_webkit_test_max_locked_shards = os.environ["WEBKIT_TEST_MAX_LOCKED_SHARDS"]
del os.environ["WEBKIT_TEST_MAX_LOCKED_SHARDS"]
_, regular_output, _ = logging_run(['--debug-rwt-logging', '--child-processes', '2'], shared_port=False)
try:
self.assertTrue(any(['1 locked' in line for line in regular_output.buflist]))
finally:
if save_env_webkit_test_max_locked_shards:
os.environ["WEBKIT_TEST_MAX_LOCKED_SHARDS"] = save_env_webkit_test_max_locked_shards
def test_child_processes_2(self):
if self.should_test_processes:
_, regular_output, _ = logging_run(
['--debug-rwt-logging', '--child-processes', '2'], shared_port=False)
self.assertTrue(any(['Running 2 ' in line for line in regular_output.buflist]))
def test_child_processes_min(self):
if self.should_test_processes:
_, regular_output, _ = logging_run(
['--debug-rwt-logging', '--child-processes', '2', '-i', 'passes/passes', 'passes'],
tests_included=True, shared_port=False)
self.assertTrue(any(['Running 1 ' in line for line in regular_output.buflist]))
def test_dryrun(self):
tests_run = get_tests_run(['--dry-run'])
self.assertEqual(tests_run, [])
tests_run = get_tests_run(['-n'])
self.assertEqual(tests_run, [])
def test_exception_raised(self):
# Exceptions raised by a worker are treated differently depending on
# whether they are in-process or out. inline exceptions work as normal,
# which allows us to get the full stack trace and traceback from the
# worker. The downside to this is that it could be any error, but this
# is actually useful in testing.
#
# Exceptions raised in a separate process are re-packaged into
# WorkerExceptions (a subclass of BaseException), which have a string capture of the stack which can
# be printed, but don't display properly in the unit test exception handlers.
self.assertRaises(BaseException, logging_run,
['failures/expected/exception.html', '--child-processes', '1'], tests_included=True)
if self.should_test_processes:
self.assertRaises(BaseException, logging_run,
['--child-processes', '2', '--force', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True, shared_port=False)
def test_full_results_html(self):
# FIXME: verify html?
details, _, _ = logging_run(['--full-results-html'])
self.assertEqual(details.exit_code, 0)
def test_hung_thread(self):
details, err, _ = logging_run(['--run-singly', '--time-out-ms=50', 'failures/expected/hang.html'], tests_included=True)
# Note that hang.html is marked as WontFix and all WontFix tests are
# expected to Pass, so that actually running them generates an "unexpected" error.
self.assertEqual(details.exit_code, 1)
self.assertNotEmpty(err)
def test_keyboard_interrupt(self):
# Note that this also tests running a test marked as SKIP if
# you specify it explicitly.
self.assertRaises(KeyboardInterrupt, logging_run, ['failures/expected/keyboard.html', '--child-processes', '1'], tests_included=True)
if self.should_test_processes:
self.assertRaises(KeyboardInterrupt, logging_run,
['failures/expected/keyboard.html', 'passes/text.html', '--child-processes', '2', '--force'], tests_included=True, shared_port=False)
def test_no_tests_found(self):
details, err, _ = logging_run(['resources'], tests_included=True)
self.assertEqual(details.exit_code, -1)
self.assertContains(err, 'No tests to run.\n')
def test_no_tests_found_2(self):
details, err, _ = logging_run(['foo'], tests_included=True)
self.assertEqual(details.exit_code, -1)
self.assertContains(err, 'No tests to run.\n')
def test_natural_order(self):
tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
tests_run = get_tests_run(['--order=natural'] + tests_to_run)
self.assertEqual(['failures/expected/missing_text.html', 'failures/expected/text.html', 'passes/args.html', 'passes/audio.html'], tests_run)
def test_natural_order_test_specified_multiple_times(self):
tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
tests_run = get_tests_run(['--order=natural'] + tests_to_run)
self.assertEqual(['passes/args.html', 'passes/args.html', 'passes/audio.html', 'passes/audio.html'], tests_run)
def test_random_order(self):
tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
tests_run = get_tests_run(['--order=random'] + tests_to_run)
self.assertEqual(sorted(tests_to_run), sorted(tests_run))
def test_random_order_test_specified_multiple_times(self):
tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
tests_run = get_tests_run(['--order=random'] + tests_to_run)
self.assertEqual(tests_run.count('passes/audio.html'), 2)
self.assertEqual(tests_run.count('passes/args.html'), 2)
def test_no_order(self):
tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
tests_run = get_tests_run(['--order=none'] + tests_to_run)
self.assertEqual(tests_to_run, tests_run)
def test_no_order_test_specified_multiple_times(self):
tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
tests_run = get_tests_run(['--order=none'] + tests_to_run)
self.assertEqual(tests_to_run, tests_run)
def test_no_order_with_directory_entries_in_natural_order(self):
tests_to_run = ['http/tests/ssl', 'perf/foo', 'http/tests/passes']
tests_run = get_tests_run(['--order=none'] + tests_to_run)
self.assertEqual(tests_run, ['http/tests/ssl/text.html', 'perf/foo/test.html', 'http/tests/passes/image.html', 'http/tests/passes/text.html'])
def test_gc_between_tests(self):
self.assertTrue(passing_run(['--gc-between-tests']))
def test_complex_text(self):
self.assertTrue(passing_run(['--complex-text']))
def test_threaded(self):
self.assertTrue(passing_run(['--threaded']))
def test_repeat_each(self):
tests_to_run = ['passes/image.html', 'passes/text.html']
tests_run = get_tests_run(['--repeat-each', '2'] + tests_to_run)
self.assertEqual(tests_run, ['passes/image.html', 'passes/image.html', 'passes/text.html', 'passes/text.html'])
def test_ignore_flag(self):
# Note that passes/image.html is expected to be run since we specified it directly.
tests_run = get_tests_run(['-i', 'passes', 'passes/image.html'])
self.assertFalse('passes/text.html' in tests_run)
self.assertTrue('passes/image.html' in tests_run)
def test_skipped_flag(self):
tests_run = get_tests_run(['passes'])
self.assertFalse('passes/skipped/skip.html' in tests_run)
num_tests_run_by_default = len(tests_run)
# Check that nothing changes when we specify skipped=default.
self.assertEqual(len(get_tests_run(['--skipped=default', 'passes'])),
num_tests_run_by_default)
# Now check that we run one more test (the skipped one).
tests_run = get_tests_run(['--skipped=ignore', 'passes'])
self.assertTrue('passes/skipped/skip.html' in tests_run)
self.assertEqual(len(tests_run), num_tests_run_by_default + 1)
# Now check that we only run the skipped test.
self.assertEqual(get_tests_run(['--skipped=only', 'passes']), ['passes/skipped/skip.html'])
# Now check that we don't run anything.
self.assertEqual(get_tests_run(['--skipped=always', 'passes/skipped/skip.html']), [])
def test_iterations(self):
tests_to_run = ['passes/image.html', 'passes/text.html']
tests_run = get_tests_run(['--iterations', '2'] + tests_to_run)
self.assertEqual(tests_run, ['passes/image.html', 'passes/text.html', 'passes/image.html', 'passes/text.html'])
def test_repeat_each_iterations_num_tests(self):
# The total number of tests should be: number_of_tests *
# repeat_each * iterations
host = MockHost()
_, err, _ = logging_run(
['--iterations', '2', '--repeat-each', '4', '--debug-rwt-logging', 'passes/text.html', 'failures/expected/text.html'],
tests_included=True, host=host)
self.assertContains(err, "All 16 tests ran as expected.\n")
def test_run_chunk(self):
# Test that we actually select the right chunk
all_tests_run = get_tests_run(['passes', 'failures'])
chunk_tests_run = get_tests_run(['--run-chunk', '1:4', 'passes', 'failures'])
self.assertEqual(all_tests_run[4:8], chunk_tests_run)
# Test that we wrap around if the number of tests is not evenly divisible by the chunk size
tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run)
self.assertEqual(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
def test_run_force(self):
# This raises an exception because we run
# failures/expected/exception.html, which is normally SKIPped.
self.assertRaises(ValueError, logging_run, ['--force'])
def test_run_part(self):
# Test that we actually select the right part
tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run)
self.assertEqual(['passes/error.html', 'passes/image.html'], tests_run)
# Test that we wrap around if the number of tests is not evenly divisible by the chunk size
# (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
# last part repeats the first two tests).
chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run)
self.assertEqual(['passes/error.html', 'passes/image.html'], chunk_tests_run)
def test_run_singly(self):
batch_tests_run = get_test_batches(['--run-singly'])
for batch in batch_tests_run:
self.assertEqual(len(batch), 1, '%s had too many tests' % ', '.join(batch))
def test_skip_failing_tests(self):
# This tests that we skip both known failing and known flaky tests. Because there are
# no known flaky tests in the default test_expectations, we add additional expectations.
host = MockHost()
host.filesystem.write_text_file('/tmp/overrides.txt', 'Bug(x) passes/image.html [ ImageOnlyFailure Pass ]\n')
batches = get_test_batches(['--skip-failing-tests', '--additional-expectations', '/tmp/overrides.txt'], host=host)
has_passes_text = False
for batch in batches:
self.assertFalse('failures/expected/text.html' in batch)
self.assertFalse('passes/image.html' in batch)
has_passes_text = has_passes_text or ('passes/text.html' in batch)
self.assertTrue(has_passes_text)
def test_run_singly_actually_runs_tests(self):
details, _, _ = logging_run(['--run-singly'], tests_included=True)
self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES - 1) # failures/expected/hang.html actually passes w/ --run-singly.
def test_single_file(self):
tests_run = get_tests_run(['passes/text.html'])
self.assertEqual(tests_run, ['passes/text.html'])
def test_single_file_with_prefix(self):
tests_run = get_tests_run(['LayoutTests/passes/text.html'])
self.assertEqual(['passes/text.html'], tests_run)
def test_single_skipped_file(self):
tests_run = get_tests_run(['failures/expected/keybaord.html'])
self.assertEqual([], tests_run)
def test_stderr_is_saved(self):
host = MockHost()
self.assertTrue(passing_run(host=host))
self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
'stuff going to stderr')
def test_test_list(self):
host = MockHost()
filename = '/tmp/foo.txt'
host.filesystem.write_text_file(filename, 'passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename], host=host)
self.assertEqual(['passes/text.html'], tests_run)
host.filesystem.remove(filename)
details, err, user = logging_run(['--test-list=%s' % filename], tests_included=True, host=host)
self.assertEqual(details.exit_code, -1)
self.assertNotEmpty(err)
def test_test_list_with_prefix(self):
host = MockHost()
filename = '/tmp/foo.txt'
host.filesystem.write_text_file(filename, 'LayoutTests/passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename], host=host)
self.assertEqual(['passes/text.html'], tests_run)
def test_missing_and_unexpected_results(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
host = MockHost()
details, err, _ = logging_run(['--no-show-results',
'failures/expected/missing_image.html',
'failures/unexpected/missing_text.html',
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host)
file_list = host.filesystem.written_files.keys()
self.assertEqual(details.exit_code, 1)
expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","image_diff_percent":1},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find(expected_token) != -1)
self.assertTrue(json_string.find('"num_regressions":1') != -1)
self.assertTrue(json_string.find('"num_flaky":0') != -1)
self.assertTrue(json_string.find('"num_missing":1') != -1)
def test_pixel_test_directories(self):
host = MockHost()
"""Both tests have faling checksum. We include only the first in pixel tests so only that should fail."""
args = ['--pixel-tests', '--pixel-test-directory', 'failures/unexpected/pixeldir',
'failures/unexpected/pixeldir/image_in_pixeldir.html',
'failures/unexpected/image_not_in_pixeldir.html']
details, err, _ = logging_run(extra_args=args, host=host, tests_included=True)
self.assertEqual(details.exit_code, 1)
expected_token = '"unexpected":{"pixeldir":{"image_in_pixeldir.html":{"expected":"PASS","actual":"IMAGE"'
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find(expected_token) != -1)
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
class CustomExitCodePort(test.TestPort):
def exit_code_from_summarized_results(self, unexpected_results):
return unexpected_results['num_regressions'] + unexpected_results['num_missing']
host = MockHost()
options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
test_port = CustomExitCodePort(host, options=options)
details, err, _ = logging_run(['--no-show-results',
'failures/expected/missing_image.html',
'failures/unexpected/missing_text.html',
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host, port_obj=test_port)
self.assertEqual(details.exit_code, 2)
def test_crash_with_stderr(self):
host = MockHost()
_, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html'], tests_included=True, host=host)
self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
def test_no_image_failure_with_image_diff(self):
host = MockHost()
_, regular_output, _ = logging_run(['failures/unexpected/checksum-with-matching-image.html'], tests_included=True, host=host)
self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1)
def test_crash_log(self):
# FIXME: Need to rewrite these tests to not be mac-specific, or move them elsewhere.
# Currently CrashLog uploading only works on Darwin.
if not self._platform.is_mac():
return
mock_crash_report = make_mock_crash_report_darwin('DumpRenderTree', 12345)
host = MockHost()
host.filesystem.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash', mock_crash_report)
_, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html'], tests_included=True, host=host)
expected_crash_log = mock_crash_report
self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/crash-with-stderr-crash-log.txt'), expected_crash_log)
def test_web_process_crash_log(self):
# FIXME: Need to rewrite these tests to not be mac-specific, or move them elsewhere.
# Currently CrashLog uploading only works on Darwin.
if not self._platform.is_mac():
return
mock_crash_report = make_mock_crash_report_darwin('WebProcess', 12345)
host = MockHost()
host.filesystem.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/WebProcess_2011-06-13-150719_quadzen.crash', mock_crash_report)
logging_run(['failures/unexpected/web-process-crash-with-stderr.html'], tests_included=True, host=host)
self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/web-process-crash-with-stderr-crash-log.txt'), mock_crash_report)
def test_exit_after_n_failures_upload(self):
host = MockHost()
details, regular_output, user = logging_run(
['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1'],
tests_included=True, host=host)
# By returning False, we know that the incremental results were generated and then deleted.
self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/incremental_results.json'))
# This checks that we report only the number of tests that actually failed.
self.assertEqual(details.exit_code, 1)
# This checks that passes/text.html is considered SKIPped.
self.assertTrue('"skipped":1' in host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json'))
# This checks that we told the user we bailed out.
self.assertTrue('Exiting early after 1 failures. 1 tests run.\n' in regular_output.getvalue())
# This checks that neither test ran as expected.
# FIXME: This log message is confusing; tests that were skipped should be called out separately.
self.assertTrue('0 tests ran as expected, 2 didn\'t:\n' in regular_output.getvalue())
def test_exit_after_n_failures(self):
# Unexpected failures should result in tests stopping.
tests_run = get_tests_run(['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1'])
self.assertEqual(['failures/unexpected/text-image-checksum.html'], tests_run)
# But we'll keep going for expected ones.
tests_run = get_tests_run(['failures/expected/text.html', 'passes/text.html', '--exit-after-n-failures', '1'])
self.assertEqual(['failures/expected/text.html', 'passes/text.html'], tests_run)
def test_exit_after_n_crashes(self):
# Unexpected crashes should result in tests stopping.
tests_run = get_tests_run(['failures/unexpected/crash.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1'])
self.assertEqual(['failures/unexpected/crash.html'], tests_run)
# Same with timeouts.
tests_run = get_tests_run(['failures/unexpected/timeout.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1'])
self.assertEqual(['failures/unexpected/timeout.html'], tests_run)
# But we'll keep going for expected ones.
tests_run = get_tests_run(['failures/expected/crash.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1'])
self.assertEqual(['failures/expected/crash.html', 'passes/text.html'], tests_run)
def test_results_directory_absolute(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
host = MockHost()
with host.filesystem.mkdtemp() as tmpdir:
_, _, user = logging_run(['--results-directory=' + str(tmpdir)], tests_included=True, host=host)
self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, host.filesystem.join(tmpdir, 'results.html'))])
def test_results_directory_default(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
# This is the default location.
_, _, user = logging_run(tests_included=True)
self.assertEqual(user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def test_results_directory_relative(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
host = MockHost()
host.filesystem.maybe_make_directory('/tmp/cwd')
host.filesystem.chdir('/tmp/cwd')
_, _, user = logging_run(['--results-directory=foo'], tests_included=True, host=host)
self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, '/tmp/cwd/foo/results.html')])
def test_retrying_and_flaky_tests(self):
host = MockHost()
details, err, _ = logging_run(['--debug-rwt-logging', 'failures/flaky'], tests_included=True, host=host)
self.assertEqual(details.exit_code, 0)
self.assertTrue('Retrying' in err.getvalue())
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/retries/failures/flaky/text-actual.txt'))
# Now we test that --clobber-old-results does remove the old entries and the old retries,
# and that we don't retry again.
host = MockHost()
details, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host)
self.assertEqual(details.exit_code, 1)
self.assertTrue('Clobbering old results' in err.getvalue())
self.assertTrue('flaky/text.html' in err.getvalue())
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
self.assertFalse(host.filesystem.exists('retries'))
def test_retrying_force_pixel_tests(self):
host = MockHost()
details, err, _ = logging_run(['--no-pixel-tests', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
self.assertEqual(details.exit_code, 1)
self.assertTrue('Retrying' in err.getvalue())
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt'))
self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.png'))
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.txt'))
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.png'))
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
json = parse_full_results(json_string)
self.assertEqual(json["tests"]["failures"]["unexpected"]["text-image-checksum.html"],
{"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1})
self.assertFalse(json["pixel_tests_enabled"])
self.assertEqual(details.enabled_pixel_tests_in_retry, True)
def test_retrying_uses_retries_directory(self):
host = MockHost()
details, err, _ = logging_run(['--debug-rwt-logging', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
self.assertEqual(details.exit_code, 1)
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt'))
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.txt'))
def test_run_order__inline(self):
# These next tests test that we run the tests in ascending alphabetical
# order per directory. HTTP tests are sharded separately from other tests,
# so we have to test both.
tests_run = get_tests_run(['-i', 'passes/passes', 'passes'])
self.assertEqual(tests_run, sorted(tests_run))
tests_run = get_tests_run(['http/tests/passes'])
self.assertEqual(tests_run, sorted(tests_run))
def test_tolerance(self):
class ImageDiffTestPort(test.TestPort):
def diff_image(self, expected_contents, actual_contents, tolerance=None):
self.tolerance_used_for_diff_image = self._options.tolerance
return (True, 1, None)
def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
host = MockHost()
test_port = ImageDiffTestPort(host, options=options)
res = passing_run(args, port_obj=test_port, tests_included=True)
self.assertTrue(res)
return test_port
base_args = ['--pixel-tests', '--no-new-test-results', 'failures/expected/*']
# If we pass in an explicit tolerance argument, then that will be used.
test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
test_port = get_port_for_run(base_args + ['--tolerance', '0'])
self.assertEqual(0, test_port.tolerance_used_for_diff_image)
# Otherwise the port's default tolerance behavior (including ignoring it)
# should be used.
test_port = get_port_for_run(base_args)
self.assertEqual(None, test_port.tolerance_used_for_diff_image)
def test_virtual(self):
self.assertTrue(passing_run(['passes/text.html', 'passes/args.html',
'virtual/passes/text.html', 'virtual/passes/args.html']))
def test_reftest_run(self):
tests_run = get_tests_run(['passes/reftest.html'])
self.assertEqual(['passes/reftest.html'], tests_run)
def test_reftest_run_reftests_if_pixel_tests_are_disabled(self):
tests_run = get_tests_run(['--no-pixel-tests', 'passes/reftest.html'])
self.assertEqual(['passes/reftest.html'], tests_run)
def test_reftest_skip_reftests_if_no_ref_tests(self):
tests_run = get_tests_run(['--no-ref-tests', 'passes/reftest.html'])
self.assertEqual([], tests_run)
tests_run = get_tests_run(['--no-ref-tests', '--no-pixel-tests', 'passes/reftest.html'])
self.assertEqual([], tests_run)
def test_reftest_expected_html_should_be_ignored(self):
tests_run = get_tests_run(['passes/reftest-expected.html'])
self.assertEqual([], tests_run)
def test_reftest_driver_should_run_expected_html(self):
tests_run = get_test_results(['passes/reftest.html'])
self.assertEqual(tests_run[0].references, ['passes/reftest-expected.html'])
def test_reftest_driver_should_run_expected_mismatch_html(self):
tests_run = get_test_results(['passes/mismatch.html'])
self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.html'])
def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
host = MockHost()
_, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find('"unlistedtest.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
self.assertTrue(json_string.find('"num_regressions":4') != -1)
self.assertTrue(json_string.find('"num_flaky":0') != -1)
self.assertTrue(json_string.find('"num_missing":1') != -1)
def test_additional_platform_directory(self):
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', '--additional-platform-directory', '/tmp/bar']))
self.assertTrue(passing_run(['--additional-platform-directory', 'foo']))
def test_additional_expectations(self):
host = MockHost()
host.filesystem.write_text_file('/tmp/overrides.txt', 'Bug(x) failures/unexpected/mismatch.html [ ImageOnlyFailure ]\n')
self.assertTrue(passing_run(['--additional-expectations', '/tmp/overrides.txt', 'failures/unexpected/mismatch.html'],
tests_included=True, host=host))
def test_no_http_and_force(self):
# See test_run_force, using --force raises an exception.
# FIXME: We would like to check the warnings generated.
self.assertRaises(ValueError, logging_run, ['--force', '--no-http'])
@staticmethod
def has_test_of_type(tests, type):
return [test for test in tests if type in test]
def test_no_http_tests(self):
batch_tests_dryrun = get_tests_run(['LayoutTests/http', 'websocket/'])
self.assertTrue(RunTest.has_test_of_type(batch_tests_dryrun, 'http'))
self.assertTrue(RunTest.has_test_of_type(batch_tests_dryrun, 'websocket'))
batch_tests_run_no_http = get_tests_run(['--no-http', 'LayoutTests/http', 'websocket/'])
self.assertFalse(RunTest.has_test_of_type(batch_tests_run_no_http, 'http'))
self.assertFalse(RunTest.has_test_of_type(batch_tests_run_no_http, 'websocket'))
batch_tests_run_http = get_tests_run(['--http', 'LayoutTests/http', 'websocket/'])
self.assertTrue(RunTest.has_test_of_type(batch_tests_run_http, 'http'))
self.assertTrue(RunTest.has_test_of_type(batch_tests_run_http, 'websocket'))
def test_platform_tests_are_found(self):
tests_run = get_tests_run(['--platform', 'test-mac-leopard', 'http'])
self.assertTrue('platform/test-mac-leopard/http/test.html' in tests_run)
self.assertFalse('platform/test-win-win7/http/test.html' in tests_run)
def test_output_diffs(self):
# Test to ensure that we don't generate -wdiff.html or -pretty.html if wdiff and PrettyPatch
# aren't available.
host = MockHost()
_, err, _ = logging_run(['--pixel-tests', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
written_files = host.filesystem.written_files
self.assertTrue(any(path.endswith('-diff.txt') for path in written_files.keys()))
self.assertFalse(any(path.endswith('-wdiff.html') for path in written_files.keys()))
self.assertFalse(any(path.endswith('-pretty-diff.html') for path in written_files.keys()))
full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
full_results = json.loads(full_results_text.replace("ADD_RESULTS(", "").replace(");", ""))
self.assertEqual(full_results['has_wdiff'], False)
self.assertEqual(full_results['has_pretty_patch'], False)
def test_unsupported_platform(self):
stdout = StringIO.StringIO()
stderr = StringIO.StringIO()
res = run_webkit_tests.main(['--platform', 'foo'], stdout, stderr)
self.assertEqual(res, run_webkit_tests.EXCEPTIONAL_EXIT_STATUS)
self.assertEqual(stdout.getvalue(), '')
self.assertTrue('unsupported platform' in stderr.getvalue())
def test_build_check(self):
# By using a port_name for a different platform than the one we're running on, the build check should always fail.
if sys.platform == 'darwin':
port_name = 'gtk'
else:
port_name = 'mac-lion'
out = StringIO.StringIO()
err = StringIO.StringIO()
self.assertEqual(run_webkit_tests.main(['--platform', port_name, 'fast/harness/results.html'], out, err), -1)
def test_verbose_in_child_processes(self):
# When we actually run multiple processes, we may have to reconfigure logging in the
# child process (e.g., on win32) and we need to make sure that works and we still
# see the verbose log output. However, we can't use logging_run() because using
# outputcapture to capture stdout and stderr latter results in a nonpicklable host.
# Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559
if not self.should_test_processes:
return
options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False)
host = MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
logging_stream = StringIO.StringIO()
run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
self.assertTrue('text.html passed' in logging_stream.getvalue())
self.assertTrue('image.html passed' in logging_stream.getvalue())
class MainTest(unittest.TestCase, StreamTestingMixin):
def setUp(self):
# A real PlatformInfo object is used here instead of a
# MockPlatformInfo because we need to actually check for
# Windows and Mac to skip some tests.
self._platform = SystemHost().platform
# FIXME: Remove this when we fix test-webkitpy to work
# properly on cygwin (bug 63846).
self.should_test_processes = not self._platform.is_win()
def test_accelerated_compositing(self):
# This just tests that we recognize the command line args
self.assertTrue(passing_run(['--accelerated-video']))
self.assertTrue(passing_run(['--no-accelerated-video']))
def test_accelerated_2d_canvas(self):
# This just tests that we recognize the command line args
self.assertTrue(passing_run(['--accelerated-2d-canvas']))
self.assertTrue(passing_run(['--no-accelerated-2d-canvas']))
def test_all(self):
res, out, err, user = logging_run([], tests_included=True)
self.assertEquals(res, unexpected_tests_count)
def test_basic(self):
self.assertTrue(passing_run())
def test_batch_size(self):
batch_tests_run = get_tests_run(['--batch-size', '2'])
for batch in batch_tests_run:
self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
def test_child_processes_2(self):
if self.should_test_processes:
_, _, regular_output, _ = logging_run(
['--print', 'config', '--child-processes', '2'])
self.assertTrue(any(['Running 2 ' in line for line in regular_output.buflist]))
def test_child_processes_min(self):
if self.should_test_processes:
_, _, regular_output, _ = logging_run(
['--print', 'config', '--child-processes', '2', 'passes'],
tests_included=True)
self.assertTrue(any(['Running 1 ' in line for line in regular_output.buflist]))
def test_dryrun(self):
batch_tests_run = get_tests_run(['--dry-run'])
self.assertEqual(batch_tests_run, [])
batch_tests_run = get_tests_run(['-n'])
self.assertEqual(batch_tests_run, [])
def test_exception_raised(self):
# Exceptions raised by a worker are treated differently depending on
# whether they are in-process or out. inline exceptions work as normal,
# which allows us to get the full stack trace and traceback from the
# worker. The downside to this is that it could be any error, but this
# is actually useful in testing.
#
# Exceptions raised in a separate process are re-packaged into
# WorkerExceptions, which have a string capture of the stack which can
# be printed, but don't display properly in the unit test exception handlers.
self.assertRaises(ValueError, logging_run,
['failures/expected/exception.html', '--child-processes', '1'], tests_included=True)
if self.should_test_processes:
self.assertRaises(run_webkit_tests.WorkerException, logging_run,
['--child-processes', '2', '--force', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True)
def test_full_results_html(self):
# FIXME: verify html?
res, out, err, user = logging_run(['--full-results-html'])
self.assertEqual(res, 0)
def test_help_printing(self):
res, out, err, user = logging_run(['--help-printing'])
self.assertEqual(res, 0)
self.assertEmpty(out)
self.assertNotEmpty(err)
def test_hung_thread(self):
res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50',
'failures/expected/hang.html'],
tests_included=True)
self.assertEqual(res, 0)
self.assertNotEmpty(out)
self.assertNotEmpty(err)
def test_keyboard_interrupt(self):
# Note that this also tests running a test marked as SKIP if
# you specify it explicitly.
self.assertRaises(KeyboardInterrupt, logging_run,
['failures/expected/keyboard.html', '--child-processes', '1'],
tests_included=True)
if self.should_test_processes:
self.assertRaises(KeyboardInterrupt, logging_run,
['failures/expected/keyboard.html', 'passes/text.html', '--child-processes', '2', '--force'], tests_included=True)
def test_no_tests_found(self):
res, out, err, user = logging_run(['resources'], tests_included=True)
self.assertEqual(res, -1)
self.assertEmpty(out)
self.assertContainsLine(err, 'No tests to run.\n')
def test_no_tests_found_2(self):
res, out, err, user = logging_run(['foo'], tests_included=True)
self.assertEqual(res, -1)
self.assertEmpty(out)
self.assertContainsLine(err, 'No tests to run.\n')
def test_randomize_order(self):
# FIXME: verify order was shuffled
self.assertTrue(passing_run(['--randomize-order']))
def test_gc_between_tests(self):
self.assertTrue(passing_run(['--gc-between-tests']))
def test_complex_text(self):
self.assertTrue(passing_run(['--complex-text']))
def test_threaded(self):
self.assertTrue(passing_run(['--threaded']))
def test_repeat_each(self):
tests_to_run = ['passes/image.html', 'passes/text.html']
tests_run = get_tests_run(['--repeat-each', '2'] + tests_to_run, tests_included=True, flatten_batches=True)
self.assertEquals(tests_run, ['passes/image.html', 'passes/image.html', 'passes/text.html', 'passes/text.html'])
def test_skip_pixel_test_if_no_baseline_option(self):
tests_to_run = ['passes/image.html', 'passes/text.html']
tests_run = get_tests_run(['--skip-pixel-test-if-no-baseline'] + tests_to_run, tests_included=True, flatten_batches=True)
self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html'])
def test_ignore_flag(self):
# Note that passes/image.html is expected to be run since we specified it directly.
tests_run = get_tests_run(['-i', 'passes', 'passes/image.html'], flatten_batches=True, tests_included=True)
self.assertFalse('passes/text.html' in tests_run)
self.assertTrue('passes/image.html' in tests_run)
def test_skipped_flag(self):
tests_run = get_tests_run(['passes'], tests_included=True, flatten_batches=True)
self.assertFalse('passes/skipped/skip.html' in tests_run)
num_tests_run_by_default = len(tests_run)
# Check that nothing changes when we specify skipped=default.
self.assertEquals(len(get_tests_run(['--skipped=default', 'passes'], tests_included=True, flatten_batches=True)),
num_tests_run_by_default)
# Now check that we run one more test (the skipped one).
tests_run = get_tests_run(['--skipped=ignore', 'passes'], tests_included=True, flatten_batches=True)
self.assertTrue('passes/skipped/skip.html' in tests_run)
self.assertEquals(len(tests_run), num_tests_run_by_default + 1)
# Now check that we only run the skipped test.
self.assertEquals(get_tests_run(['--skipped=only', 'passes'], tests_included=True, flatten_batches=True),
['passes/skipped/skip.html'])
def test_iterations(self):
tests_to_run = ['passes/image.html', 'passes/text.html']
tests_run = get_tests_run(['--iterations', '2'] + tests_to_run, tests_included=True, flatten_batches=True)
self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html', 'passes/image.html', 'passes/text.html'])
def test_repeat_each_iterations_num_tests(self):
# The total number of tests should be: number_of_tests *
# repeat_each * iterations
host = MockHost()
res, out, err, _ = logging_run(['--iterations', '2',
'--repeat-each', '4',
'--print', 'everything',
'passes/text.html', 'failures/expected/text.html'],
tests_included=True, host=host, record_results=True)
self.assertContainsLine(out, "=> Results: 8/16 tests passed (50.0%)\n")
self.assertContainsLine(err, "All 16 tests ran as expected.\n")
def test_run_chunk(self):
# Test that we actually select the right chunk
all_tests_run = get_tests_run(flatten_batches=True)
chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True)
self.assertEquals(all_tests_run[4:8], chunk_tests_run)
# Test that we wrap around if the number of tests is not evenly divisible by the chunk size
tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True)
self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
def test_run_force(self):
# This raises an exception because we run
# failures/expected/exception.html, which is normally SKIPped.
# See also the comments in test_exception_raised() about ValueError vs. WorkerException.
self.assertRaises(ValueError, logging_run, ['--force'])
def test_run_part(self):
# Test that we actually select the right part
tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True)
self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run)
# Test that we wrap around if the number of tests is not evenly divisible by the chunk size
# (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
# last part repeats the first two tests).
chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True)
self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run)
def test_run_singly(self):
batch_tests_run = get_tests_run(['--run-singly'])
for batch in batch_tests_run:
self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
def test_skip_failing_tests(self):
# This tests that we skip both known failing and known flaky tests. Because there are
# no known flaky tests in the default test_expectations, we add additional expectations.
host = MockHost()
host.filesystem.write_text_file('/tmp/overrides.txt', 'BUGX : passes/image.html = IMAGE PASS\n')
batches = get_tests_run(['--skip-failing-tests', '--additional-expectations', '/tmp/overrides.txt'], host=host)
has_passes_text = False
for batch in batches:
self.assertFalse('failures/expected/text.html' in batch)
self.assertFalse('passes/image.html' in batch)
has_passes_text = has_passes_text or ('passes/text.html' in batch)
self.assertTrue(has_passes_text)
def test_run_singly_actually_runs_tests(self):
res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
self.assertEquals(res, 8)
def test_single_file(self):
# FIXME: We should consider replacing more of the get_tests_run()-style tests
# with tests that read the tests_run* files, like this one.
host = MockHost()
tests_run = passing_run(['passes/text.html'], tests_included=True, host=host)
self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/tests_run0.txt'),
'passes/text.html\n')
def test_single_file_with_prefix(self):
tests_run = get_tests_run(['LayoutTests/passes/text.html'], tests_included=True, flatten_batches=True)
self.assertEquals(['passes/text.html'], tests_run)
def test_single_skipped_file(self):
tests_run = get_tests_run(['failures/expected/keybaord.html'], tests_included=True, flatten_batches=True)
self.assertEquals([], tests_run)
def test_stderr_is_saved(self):
host = MockHost()
self.assertTrue(passing_run(host=host))
self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
'stuff going to stderr')
def test_test_list(self):
host = MockHost()
filename = '/tmp/foo.txt'
host.filesystem.write_text_file(filename, 'passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, host=host)
self.assertEquals(['passes/text.html'], tests_run)
host.filesystem.remove(filename)
res, out, err, user = logging_run(['--test-list=%s' % filename],
tests_included=True, host=host)
self.assertEqual(res, -1)
self.assertNotEmpty(err)
def test_test_list_with_prefix(self):
host = MockHost()
filename = '/tmp/foo.txt'
host.filesystem.write_text_file(filename, 'LayoutTests/passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, host=host)
self.assertEquals(['passes/text.html'], tests_run)
def test_unexpected_failures(self):
# Run tests including the unexpected failures.
self._url_opened = None
res, out, err, user = logging_run(tests_included=True)
self.assertEqual(res, unexpected_tests_count)
self.assertNotEmpty(out)
self.assertNotEmpty(err)
self.assertEqual(user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def test_missing_and_unexpected_results(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
host = MockHost()
res, out, err, _ = logging_run(['--no-show-results',
'failures/expected/missing_image.html',
'failures/unexpected/missing_text.html',
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host, record_results=True)
file_list = host.filesystem.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEquals(res, 1)
expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find(expected_token) != -1)
self.assertTrue(json_string.find('"num_regressions":1') != -1)
self.assertTrue(json_string.find('"num_flaky":0') != -1)
self.assertTrue(json_string.find('"num_missing":1') != -1)
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
class CustomExitCodePort(TestPort):
def exit_code_from_summarized_results(self, unexpected_results):
return unexpected_results['num_regressions'] + unexpected_results['num_missing']
host = MockHost()
options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
test_port = CustomExitCodePort(host, options=options)
res, out, err, _ = logging_run(['--no-show-results',
'failures/expected/missing_image.html',
'failures/unexpected/missing_text.html',
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host, record_results=True, port_obj=test_port)
self.assertEquals(res, 2)
def test_crash_with_stderr(self):
host = MockHost()
res, buildbot_output, regular_output, user = logging_run([
'failures/unexpected/crash-with-stderr.html',
],
tests_included=True,
record_results=True,
host=host)
self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
def test_no_image_failure_with_image_diff(self):
host = MockHost()
res, buildbot_output, regular_output, user = logging_run([
'failures/unexpected/checksum-with-matching-image.html',
],
tests_included=True,
record_results=True,
host=host)
self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1)
def test_crash_log(self):
# FIXME: Need to rewrite these tests to not be mac-specific, or move them elsewhere.
# Currently CrashLog uploading only works on Darwin.
if not self._platform.is_mac():
return
mock_crash_report = make_mock_crash_report_darwin('DumpRenderTree', 12345)
host = MockHost()
host.filesystem.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash', mock_crash_report)
res, buildbot_output, regular_output, user = logging_run([
'failures/unexpected/crash-with-stderr.html',
],
tests_included=True,
record_results=True,
host=host)
expected_crash_log = mock_crash_report
self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/crash-with-stderr-crash-log.txt'), expected_crash_log)
def test_web_process_crash_log(self):
# FIXME: Need to rewrite these tests to not be mac-specific, or move them elsewhere.
# Currently CrashLog uploading only works on Darwin.
if not self._platform.is_mac():
return
mock_crash_report = make_mock_crash_report_darwin('WebProcess', 12345)
host = MockHost()
host.filesystem.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/WebProcess_2011-06-13-150719_quadzen.crash', mock_crash_report)
res, buildbot_output, regular_output, user = logging_run([
'failures/unexpected/web-process-crash-with-stderr.html',
],
tests_included=True,
record_results=True,
host=host)
self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/web-process-crash-with-stderr-crash-log.txt'), mock_crash_report)
def test_exit_after_n_failures_upload(self):
host = MockHost()
res, buildbot_output, regular_output, user = logging_run([
'failures/unexpected/text-image-checksum.html',
'passes/text.html',
'--exit-after-n-failures', '1',
],
tests_included=True,
record_results=True,
host=host)
# By returning False, we know that the incremental results were generated and then deleted.
self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/incremental_results.json'))
# This checks that we report only the number of tests that actually failed.
self.assertEquals(res, 1)
# This checks that passes/text.html is considered SKIPped.
self.assertTrue('"skipped":1' in host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json'))
# This checks that we told the user we bailed out.
self.assertTrue('Exiting early after 1 failures. 1 tests run.\n' in regular_output.getvalue())
# This checks that neither test ran as expected.
# FIXME: This log message is confusing; tests that were skipped should be called out separately.
self.assertTrue('0 tests ran as expected, 2 didn\'t:\n' in regular_output.getvalue())
def test_exit_after_n_failures(self):
# Unexpected failures should result in tests stopping.
tests_run = get_tests_run([
'failures/unexpected/text-image-checksum.html',
'passes/text.html',
'--exit-after-n-failures', '1',
],
tests_included=True,
flatten_batches=True)
self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run)
# But we'll keep going for expected ones.
tests_run = get_tests_run([
'failures/expected/text.html',
'passes/text.html',
'--exit-after-n-failures', '1',
],
tests_included=True,
flatten_batches=True)
self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run)
def test_exit_after_n_crashes(self):
# Unexpected crashes should result in tests stopping.
tests_run = get_tests_run([
'failures/unexpected/crash.html',
'passes/text.html',
'--exit-after-n-crashes-or-timeouts', '1',
],
tests_included=True,
flatten_batches=True)
self.assertEquals(['failures/unexpected/crash.html'], tests_run)
# Same with timeouts.
tests_run = get_tests_run([
'failures/unexpected/timeout.html',
'passes/text.html',
'--exit-after-n-crashes-or-timeouts', '1',
],
tests_included=True,
flatten_batches=True)
self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
# But we'll keep going for expected ones.
tests_run = get_tests_run([
'failures/expected/crash.html',
'passes/text.html',
'--exit-after-n-crashes-or-timeouts', '1',
],
tests_included=True,
flatten_batches=True)
self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run)
def test_results_directory_absolute(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
host = MockHost()
with host.filesystem.mkdtemp() as tmpdir:
res, out, err, user = logging_run(['--results-directory=' + str(tmpdir)],
tests_included=True, host=host)
self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, host.filesystem.join(tmpdir, 'results.html'))])
def test_results_directory_default(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
# This is the default location.
res, out, err, user = logging_run(tests_included=True)
self.assertEqual(user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def test_results_directory_relative(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
host = MockHost()
host.filesystem.maybe_make_directory('/tmp/cwd')
host.filesystem.chdir('/tmp/cwd')
res, out, err, user = logging_run(['--results-directory=foo'],
tests_included=True, host=host)
self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, '/tmp/cwd/foo/results.html')])
def test_retrying_and_flaky_tests(self):
host = MockHost()
res, out, err, _ = logging_run(['failures/flaky'], tests_included=True, host=host)
self.assertEquals(res, 0)
self.assertTrue('Retrying' in err.getvalue())
self.assertTrue('Unexpected flakiness' in out.getvalue())
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/tests_run0.txt'))
self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/retries/failures/flaky/text-actual.txt'))
# Now we test that --clobber-old-results does remove the old entries and the old retries,
# and that we don't retry again.
res, out, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host)
self.assertEquals(res, 1)
self.assertTrue('Clobbering old results' in err.getvalue())
self.assertTrue('flaky/text.html' in err.getvalue())
self.assertTrue('Unexpected text diff' in out.getvalue())
self.assertFalse('Unexpected flakiness' in out.getvalue())
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
self.assertFalse(host.filesystem.exists('retries'))
# These next tests test that we run the tests in ascending alphabetical
# order per directory. HTTP tests are sharded separately from other tests,
# so we have to test both.
def assert_run_order(self, child_processes='1'):
tests_run = get_tests_run(['--child-processes', child_processes, 'passes'],
tests_included=True, flatten_batches=True)
self.assertEquals(tests_run, sorted(tests_run))
tests_run = get_tests_run(['--child-processes', child_processes, 'http/tests/passes'],
tests_included=True, flatten_batches=True)
self.assertEquals(tests_run, sorted(tests_run))
def test_run_order__inline(self):
self.assert_run_order()
def test_tolerance(self):
class ImageDiffTestPort(TestPort):
def diff_image(self, expected_contents, actual_contents, tolerance=None):
self.tolerance_used_for_diff_image = self._options.tolerance
return (True, 1)
def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
host = MockHost()
test_port = ImageDiffTestPort(host, options=options)
res = passing_run(args, port_obj=test_port, tests_included=True)
self.assertTrue(res)
return test_port
base_args = ['--pixel-tests', '--no-new-test-results', 'failures/expected/*']
# If we pass in an explicit tolerance argument, then that will be used.
test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
test_port = get_port_for_run(base_args + ['--tolerance', '0'])
self.assertEqual(0, test_port.tolerance_used_for_diff_image)
# Otherwise the port's default tolerance behavior (including ignoring it)
# should be used.
test_port = get_port_for_run(base_args)
self.assertEqual(None, test_port.tolerance_used_for_diff_image)
def test_virtual(self):
self.assertTrue(passing_run(['passes/text.html', 'passes/args.html',
'virtual/passes/text.html', 'virtual/passes/args.html']))
def test_reftest_run(self):
tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True)
self.assertEquals(['passes/reftest.html'], tests_run)
def test_reftest_run_reftests_if_pixel_tests_are_disabled(self):
tests_run = get_tests_run(['--no-pixel-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
self.assertEquals(['passes/reftest.html'], tests_run)
def test_reftest_skip_reftests_if_no_ref_tests(self):
tests_run = get_tests_run(['--no-ref-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
self.assertEquals([], tests_run)
tests_run = get_tests_run(['--no-ref-tests', '--no-pixel-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
self.assertEquals([], tests_run)
def test_reftest_expected_html_should_be_ignored(self):
tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True)
self.assertEquals([], tests_run)
def test_reftest_driver_should_run_expected_html(self):
tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True, include_reference_html=True)
self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run)
def test_reftest_driver_should_run_expected_mismatch_html(self):
tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True, include_reference_html=True)
self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run)
def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
host = MockHost()
res, out, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host, record_results=True)
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find('"unlistedtest.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
self.assertTrue(json_string.find('"num_regressions":4') != -1)
self.assertTrue(json_string.find('"num_flaky":0') != -1)
self.assertTrue(json_string.find('"num_missing":1') != -1)
def test_additional_platform_directory(self):
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', '--additional-platform-directory', '/tmp/bar']))
res, buildbot_output, regular_output, user = logging_run(['--additional-platform-directory', 'foo'])
self.assertContainsLine(regular_output, '--additional-platform-directory=foo is ignored since it is not absolute\n')
def test_additional_expectations(self):
host = MockHost()
host.filesystem.write_text_file('/tmp/overrides.txt', 'BUGX : failures/unexpected/mismatch.html = IMAGE\n')
self.assertTrue(passing_run(['--additional-expectations', '/tmp/overrides.txt', 'failures/unexpected/mismatch.html'],
tests_included=True, host=host))
def test_no_http_and_force(self):
# See test_run_force, using --force raises an exception.
# FIXME: We would like to check the warnings generated.
self.assertRaises(ValueError, logging_run, ['--force', '--no-http'])
@staticmethod
def has_test_of_type(tests, type):
return [test for test in tests if type in test]
def test_no_http_tests(self):
batch_tests_dryrun = get_tests_run(['LayoutTests/http', 'websocket/'], flatten_batches=True)
self.assertTrue(MainTest.has_test_of_type(batch_tests_dryrun, 'http'))
self.assertTrue(MainTest.has_test_of_type(batch_tests_dryrun, 'websocket'))
batch_tests_run_no_http = get_tests_run(['--no-http', 'LayoutTests/http', 'websocket/'], flatten_batches=True)
self.assertFalse(MainTest.has_test_of_type(batch_tests_run_no_http, 'http'))
self.assertFalse(MainTest.has_test_of_type(batch_tests_run_no_http, 'websocket'))
batch_tests_run_http = get_tests_run(['--http', 'LayoutTests/http', 'websocket/'], flatten_batches=True)
self.assertTrue(MainTest.has_test_of_type(batch_tests_run_http, 'http'))
self.assertTrue(MainTest.has_test_of_type(batch_tests_run_http, 'websocket'))
def test_platform_tests_are_found(self):
tests_run = get_tests_run(['http'], tests_included=True, flatten_batches=True)
self.assertTrue('platform/test-snow-leopard/http/test.html' in tests_run)
def test_output_diffs(self):
# Test to ensure that we don't generate -wdiff.html or -pretty.html if wdiff and PrettyPatch
# aren't available.
host = MockHost()
res, out, err, _ = logging_run(['--pixel-tests', 'failures/unexpected/text-image-checksum.html'],
tests_included=True, record_results=True, host=host)
written_files = host.filesystem.written_files
self.assertTrue(any(path.endswith('-diff.txt') for path in written_files.keys()))
self.assertFalse(any(path.endswith('-wdiff.html') for path in written_files.keys()))
self.assertFalse(any(path.endswith('-pretty-diff.html') for path in written_files.keys()))
full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
full_results = json.loads(full_results_text.replace("ADD_RESULTS(", "").replace(");", ""))
self.assertEquals(full_results['has_wdiff'], False)
self.assertEquals(full_results['has_pretty_patch'], False)
