def __init__(self, running_port, platform, options):
self._platform = platform
self._options = options
# This is the port that the script is running on.
self._port = running_port
# This is the port that the rebaselining tool creates new baselines for.
self._rebaseline_port = port.get(
self._port.test_platform_name_to_name(platform), options)
self._rebaselining_tests = []
self._rebaselined_tests = []
# Create tests and expectations helper which is used to:
# -. compile list of tests that need rebaselining.
# -. update the tests in test_expectations file after rebaseline
# is done.
expectations_str = self._rebaseline_port.test_expectations()
self._test_expectations = \
test_expectations.TestExpectations(self._rebaseline_port,
None,
expectations_str,
self._platform,
False,
False)
self._repo_type = self._get_repo_type()
def __init__(self, running_port, target_port, platform, options):
"""
Args:
running_port: the Port the script is running on.
target_port: the Port the script uses to find port-specific
configuration information like the test_expectations.txt
file location and the list of test platforms.
platform: the test platform to rebaseline
options: the command-line options object."""
self._platform = platform
self._options = options
self._port = running_port
self._target_port = target_port
self._rebaseline_port = port.get(self._target_port.test_platform_name_to_name(platform), options)
self._rebaselining_tests = []
self._rebaselined_tests = []
# Create tests and expectations helper which is used to:
# -. compile list of tests that need rebaselining.
# -. update the tests in test_expectations file after rebaseline
# is done.
expectations_str = self._rebaseline_port.test_expectations()
self._test_expectations = test_expectations.TestExpectations(
self._rebaseline_port, None, expectations_str, self._platform, False, False
)
self._scm = scm.default_scm()
def __init__(self, running_port, target_port, platform, options):
"""
Args:
running_port: the Port the script is running on.
target_port: the Port the script uses to find port-specific
configuration information like the test_expectations.txt
file location and the list of test platforms.
platform: the test platform to rebaseline
options: the command-line options object."""
self._platform = platform
self._options = options
self._port = running_port
self._target_port = target_port
self._rebaseline_port = port.get(
self._target_port.test_platform_name_to_name(platform), options)
self._rebaselining_tests = []
self._rebaselined_tests = []
# Create tests and expectations helper which is used to:
# -. compile list of tests that need rebaselining.
# -. update the tests in test_expectations file after rebaseline
# is done.
expectations_str = self._rebaseline_port.test_expectations()
self._test_expectations = \
test_expectations.TestExpectations(self._rebaseline_port,
None,
expectations_str,
self._platform,
False,
False)
self._scm = scm.default_scm()
def get_host_port_object(options):
"""Return a port object for the platform we're running on."""
# The only thing we really need on the host is a way to diff
# text files and image files, which means we need to check that some
# version of ImageDiff has been built. We will look for either Debug
# or Release versions of the default port on the platform.
options.configuration = "Release"
port_obj = port.get(None, options)
if not port_obj.check_image_diff(override_step=None, logging=False):
_log.debug("No release version of the image diff binary was found.")
options.configuration = "Debug"
port_obj = port.get(None, options)
if not port_obj.check_image_diff(override_step=None, logging=False):
_log.error("No version of image diff was found. Check your build.")
return None
else:
_log.debug("Found the debug version of the image diff binary.")
else:
_log.debug("Found the release version of the image diff binary.")
return port_obj
def get_host_port_object(options):
"""Return a port object for the platform we're running on."""
# The only thing we really need on the host is a way to diff
# text files and image files, which means we need to check that some
# version of ImageDiff has been built. We will look for either Debug
# or Release versions of the default port on the platform.
options.configuration = "Release"
port_obj = port.get(None, options)
if not port_obj.check_image_diff(override_step=None, logging=False):
_log.debug('No release version of the image diff binary was found.')
options.configuration = "Debug"
port_obj = port.get(None, options)
if not port_obj.check_image_diff(override_step=None, logging=False):
_log.error('No version of image diff was found. Check your build.')
return None
else:
_log.debug('Found the debug version of the image diff binary.')
else:
_log.debug('Found the release version of the image diff binary.')
return port_obj
def main(executive=Executive()):
"""Main function to produce new baselines."""
option_parser = optparse.OptionParser()
option_parser.add_option("-v", "--verbose", action="store_true", default=False, help="include debug-level logging.")
option_parser.add_option("-q", "--quiet", action="store_true", help="Suppress result HTML viewing")
option_parser.add_option(
"-p",
"--platforms",
default="mac,win,win-xp,win-vista,linux",
help=("Comma delimited list of platforms " "that need rebaselining."),
)
option_parser.add_option(
"-u",
"--archive_url",
default=("http://build.chromium.org/buildbot/" "layout_test_results"),
help=("Url to find the layout test result archive" " file."),
)
option_parser.add_option(
"-U", "--force_archive_url", help=("Url of result zip file. This option is for debugging " "purposes")
)
option_parser.add_option(
"-w",
"--webkit_canary",
action="store_true",
default=False,
help=("If True, pull baselines from webkit.org " "canary bot."),
)
option_parser.add_option(
"-b",
"--backup",
action="store_true",
default=False,
help=("Whether or not to backup the original test" " expectations file after rebaseline."),
)
option_parser.add_option(
"-d",
"--html_directory",
default="",
help=("The directory that stores the results for " "rebaselining comparison."),
)
option_parser.add_option(
"",
"--use_drt",
action="store_true",
default=False,
help=("Use ImageDiff from DumpRenderTree instead " "of image_diff for pixel tests."),
)
option_parser.add_option(
"",
"--target-platform",
default="chromium",
help=("The target platform to rebaseline " '("mac", "chromium", "qt", etc.). Defaults ' 'to "chromium".'),
)
options = option_parser.parse_args()[0]
# We need to create three different Port objects over the life of this
# script. |target_port_obj| is used to determine configuration information:
# location of the expectations file, names of ports to rebaseline, etc.
# |port_obj| is used for runtime functionality like actually diffing
# Then we create a rebaselining port to actual find and manage the
# baselines.
target_options = copy.copy(options)
if options.target_platform == "chromium":
target_options.chromium = True
target_port_obj = port.get(None, target_options)
# Set up our logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(
level=log_level,
format=("%(asctime)s %(filename)s:%(lineno)-3d " "%(levelname)s %(message)s"),
datefmt="%y%m%d %H:%M:%S",
)
host_port_obj = get_host_port_object(options)
if not host_port_obj:
sys.exit(1)
# Verify 'platforms' option is valid.
if not options.platforms:
_log.error('Invalid "platforms" option. --platforms must be ' "specified in order to rebaseline.")
sys.exit(1)
platforms = [p.strip().lower() for p in options.platforms.split(",")]
for platform in platforms:
if not platform in REBASELINE_PLATFORM_ORDER:
_log.error('Invalid platform: "%s"' % (platform))
sys.exit(1)
# Adjust the platform order so rebaseline tool is running at the order of
# 'mac', 'win' and 'linux'. This is in same order with layout test baseline
# search paths. It simplifies how the rebaseline tool detects duplicate
# baselines. Check _IsDupBaseline method for details.
rebaseline_platforms = []
for platform in REBASELINE_PLATFORM_ORDER:
if platform in platforms:
rebaseline_platforms.append(platform)
options.html_directory = setup_html_directory(options.html_directory)
rebaselining_tests = set()
backup = options.backup
for platform in rebaseline_platforms:
rebaseliner = Rebaseliner(host_port_obj, target_port_obj, platform, options)
_log.info("")
log_dashed_string("Rebaseline started", platform)
if rebaseliner.run(backup):
# Only need to backup one original copy of test expectation file.
backup = False
log_dashed_string("Rebaseline done", platform)
else:
log_dashed_string("Rebaseline failed", platform, logging.ERROR)
rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
_log.info("")
log_dashed_string("Rebaselining result comparison started", None)
html_generator = HtmlGenerator(
target_port_obj, options, rebaseline_platforms, rebaselining_tests, executive=executive
)
html_generator.generate_html()
if not options.quiet:
html_generator.show_html()
log_dashed_string("Rebaselining result comparison done", None)
sys.exit(0)
def main():
options, args = parse_args()
port_obj = port.get(options.platform, options)
return run(port_obj, options, args)
def main():
"""Main function to produce new baselines."""
option_parser = optparse.OptionParser()
option_parser.add_option('-v', '--verbose',
action='store_true',
default=False,
help='include debug-level logging.')
option_parser.add_option('-p', '--platforms',
default='mac,win,win-xp,win-vista,linux',
help=('Comma delimited list of platforms '
'that need rebaselining.'))
option_parser.add_option('-u', '--archive_url',
default=('http://build.chromium.org/buildbot/'
'layout_test_results'),
help=('Url to find the layout test result archive'
' file.'))
option_parser.add_option('-w', '--webkit_canary',
action='store_true',
default=False,
help=('If True, pull baselines from webkit.org '
'canary bot.'))
option_parser.add_option('-b', '--backup',
action='store_true',
default=False,
help=('Whether or not to backup the original test'
' expectations file after rebaseline.'))
option_parser.add_option('-d', '--html_directory',
default='',
help=('The directory that stores the results for'
' rebaselining comparison.'))
option_parser.add_option('', '--target-platform',
default='chromium',
help=('The target platform to rebaseline '
'("mac", "chromium", "qt", etc.). Defaults '
'to "chromium".'))
options = option_parser.parse_args()[0]
# We need to create three different Port objects over the life of this
# script. |target_port_obj| is used to determine configuration information:
# location of the expectations file, names of ports to rebaseline, etc.
# |port_obj| is used for runtime functionality like actually diffing
# Then we create a rebaselining port to actual find and manage the
# baselines.
target_options = copy.copy(options)
if options.target_platform == 'chromium':
target_options.chromium = True
target_port_obj = port.get(None, target_options)
# Set up our logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format=('%(asctime)s %(filename)s:%(lineno)-3d '
'%(levelname)s %(message)s'),
datefmt='%y%m%d %H:%M:%S')
host_port_obj = get_host_port_object(options)
if not host_port_obj:
sys.exit(1)
# Verify 'platforms' option is valid.
if not options.platforms:
_log.error('Invalid "platforms" option. --platforms must be '
'specified in order to rebaseline.')
sys.exit(1)
platforms = [p.strip().lower() for p in options.platforms.split(',')]
for platform in platforms:
if not platform in REBASELINE_PLATFORM_ORDER:
_log.error('Invalid platform: "%s"' % (platform))
sys.exit(1)
# Adjust the platform order so rebaseline tool is running at the order of
# 'mac', 'win' and 'linux'. This is in same order with layout test baseline
# search paths. It simplifies how the rebaseline tool detects duplicate
# baselines. Check _IsDupBaseline method for details.
rebaseline_platforms = []
for platform in REBASELINE_PLATFORM_ORDER:
if platform in platforms:
rebaseline_platforms.append(platform)
options.html_directory = setup_html_directory(options.html_directory)
rebaselining_tests = set()
backup = options.backup
for platform in rebaseline_platforms:
rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
platform, options)
_log.info('')
log_dashed_string('Rebaseline started', platform)
if rebaseliner.run(backup):
# Only need to backup one original copy of test expectation file.
backup = False
log_dashed_string('Rebaseline done', platform)
else:
log_dashed_string('Rebaseline failed', platform, logging.ERROR)
rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
_log.info('')
log_dashed_string('Rebaselining result comparison started', None)
html_generator = HtmlGenerator(target_port_obj,
options,
rebaseline_platforms,
rebaselining_tests)
html_generator.generate_html()
html_generator.show_html()
log_dashed_string('Rebaselining result comparison done', None)
sys.exit(0)
def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
self._port = port.get('test', None)
self._exp = None
unittest.TestCase.__init__(self, testFunc)
def main():
options, args = parse_args()
port_obj = port.get(options.platform, options)
return run(port_obj, options, args)
def main():
"""Main function to produce new baselines."""
option_parser = optparse.OptionParser()
option_parser.add_option('-v', '--verbose',
action='store_true',
default=False,
help='include debug-level logging.')
option_parser.add_option('-p', '--platforms',
default='mac,win,win-xp,win-vista,linux',
help=('Comma delimited list of platforms '
'that need rebaselining.'))
option_parser.add_option('-u', '--archive_url',
default=('http://build.chromium.org/buildbot/'
'layout_test_results'),
help=('Url to find the layout test result archive'
' file.'))
option_parser.add_option('-w', '--webkit_canary',
action='store_true',
default=False,
help=('If True, pull baselines from webkit.org '
'canary bot.'))
option_parser.add_option('-b', '--backup',
action='store_true',
default=False,
help=('Whether or not to backup the original test'
' expectations file after rebaseline.'))
option_parser.add_option('-d', '--html_directory',
default='',
help=('The directory that stores the results for'
' rebaselining comparison.'))
options = option_parser.parse_args()[0]
# FIXME: Setting options.chromium to True forces port.get() to only look
# at Chromium ports; we hard-code this because this script doesn't work
# on non-Chromium ports yet. We should fix that.
options.chromium = True
port_obj = port.get(None, options)
# Set up our logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format=('%(asctime)s %(filename)s:%(lineno)-3d '
'%(levelname)s %(message)s'),
datefmt='%y%m%d %H:%M:%S')
# Verify 'platforms' option is valid
if not options.platforms:
_log.error('Invalid "platforms" option. --platforms must be '
'specified in order to rebaseline.')
sys.exit(1)
platforms = [p.strip().lower() for p in options.platforms.split(',')]
for platform in platforms:
if not platform in REBASELINE_PLATFORM_ORDER:
_log.error('Invalid platform: "%s"' % (platform))
sys.exit(1)
# Adjust the platform order so rebaseline tool is running at the order of
# 'mac', 'win' and 'linux'. This is in same order with layout test baseline
# search paths. It simplifies how the rebaseline tool detects duplicate
# baselines. Check _IsDupBaseline method for details.
rebaseline_platforms = []
for platform in REBASELINE_PLATFORM_ORDER:
if platform in platforms:
rebaseline_platforms.append(platform)
options.html_directory = setup_html_directory(options.html_directory)
rebaselining_tests = set()
backup = options.backup
for platform in rebaseline_platforms:
rebaseliner = Rebaseliner(port_obj, platform, options)
_log.info('')
log_dashed_string('Rebaseline started', platform)
if rebaseliner.run(backup):
# Only need to backup one original copy of test expectation file.
backup = False
log_dashed_string('Rebaseline done', platform)
else:
log_dashed_string('Rebaseline failed', platform, logging.ERROR)
rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
_log.info('')
log_dashed_string('Rebaselining result comparison started', None)
html_generator = HtmlGenerator(port_obj,
options,
rebaseline_platforms,
rebaselining_tests)
html_generator.generate_html()
html_generator.show_html()
log_dashed_string('Rebaselining result comparison done', None)
sys.exit(0)
),
optparse.make_option(
"--release",
action="store_const",
const="Release",
dest="configuration",
help="Set the configuration to Release",
),
]
misc_options = [
optparse.make_option(
"-p",
"--platform",
action="store",
default="mac",
help='Platform to test (e.g., "mac", "chromium-mac", etc.',
),
optparse.make_option(
"--timeout", action="store", default="2000", help="test timeout in milliseconds (2000 by default)"
),
optparse.make_option("--wrapper", action="store"),
optparse.make_option(
"--no-pixel-tests", action="store_true", default=False, help="disable pixel-to-pixel PNG comparisons"
),
]
option_list = configuration_options + misc_options
optparser = optparse.OptionParser(option_list=option_list)
options, args = optparser.parse_args()
p = port.get(options.platform, options)
run_tests(p, options, args)
def main():
"""Main function to produce new baselines."""
option_parser = optparse.OptionParser()
option_parser.add_option('-v',
'--verbose',
action='store_true',
default=False,
help='include debug-level logging.')
option_parser.add_option('-q',
'--quiet',
action='store_true',
help='Suppress result HTML viewing')
option_parser.add_option('-p',
'--platforms',
default='mac,win,win-xp,win-vista,linux',
help=('Comma delimited list of platforms '
'that need rebaselining.'))
option_parser.add_option('-u',
'--archive_url',
default=('http://build.chromium.org/buildbot/'
'layout_test_results'),
help=('Url to find the layout test result archive'
' file.'))
option_parser.add_option(
'-U',
'--force_archive_url',
help=('Url of result zip file. This option is for debugging '
'purposes'))
option_parser.add_option('-w',
'--webkit_canary',
action='store_true',
default=False,
help=('If True, pull baselines from webkit.org '
'canary bot.'))
option_parser.add_option('-b',
'--backup',
action='store_true',
default=False,
help=('Whether or not to backup the original test'
' expectations file after rebaseline.'))
option_parser.add_option('-d',
'--html_directory',
default='',
help=('The directory that stores the results for '
'rebaselining comparison.'))
option_parser.add_option('',
'--use_drt',
action='store_true',
default=False,
help=('Use ImageDiff from DumpRenderTree instead '
'of image_diff for pixel tests.'))
option_parser.add_option('',
'--target-platform',
default='chromium',
help=('The target platform to rebaseline '
'("mac", "chromium", "qt", etc.). Defaults '
'to "chromium".'))
options = option_parser.parse_args()[0]
# We need to create three different Port objects over the life of this
# script. |target_port_obj| is used to determine configuration information:
# location of the expectations file, names of ports to rebaseline, etc.
# |port_obj| is used for runtime functionality like actually diffing
# Then we create a rebaselining port to actual find and manage the
# baselines.
target_options = copy.copy(options)
if options.target_platform == 'chromium':
target_options.chromium = True
target_port_obj = port.get(None, target_options)
# Set up our logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format=('%(asctime)s %(filename)s:%(lineno)-3d '
'%(levelname)s %(message)s'),
datefmt='%y%m%d %H:%M:%S')
host_port_obj = get_host_port_object(options)
if not host_port_obj:
sys.exit(1)
# Verify 'platforms' option is valid.
if not options.platforms:
_log.error('Invalid "platforms" option. --platforms must be '
'specified in order to rebaseline.')
sys.exit(1)
platforms = [p.strip().lower() for p in options.platforms.split(',')]
for platform in platforms:
if not platform in REBASELINE_PLATFORM_ORDER:
_log.error('Invalid platform: "%s"' % (platform))
sys.exit(1)
# Adjust the platform order so rebaseline tool is running at the order of
# 'mac', 'win' and 'linux'. This is in same order with layout test baseline
# search paths. It simplifies how the rebaseline tool detects duplicate
# baselines. Check _IsDupBaseline method for details.
rebaseline_platforms = []
for platform in REBASELINE_PLATFORM_ORDER:
if platform in platforms:
rebaseline_platforms.append(platform)
options.html_directory = setup_html_directory(options.html_directory)
rebaselining_tests = set()
backup = options.backup
for platform in rebaseline_platforms:
rebaseliner = Rebaseliner(host_port_obj, target_port_obj, platform,
options)
_log.info('')
log_dashed_string('Rebaseline started', platform)
if rebaseliner.run(backup):
# Only need to backup one original copy of test expectation file.
backup = False
log_dashed_string('Rebaseline done', platform)
else:
log_dashed_string('Rebaseline failed', platform, logging.ERROR)
rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
_log.info('')
log_dashed_string('Rebaselining result comparison started', None)
html_generator = HtmlGenerator(target_port_obj, options,
rebaseline_platforms, rebaselining_tests)
html_generator.generate_html()
if not options.quiet:
html_generator.show_html()
log_dashed_string('Rebaselining result comparison done', None)
sys.exit(0)
print "uri: " + uri
crash, timeout, checksum, output, err = \
driver.run_test(uri, int(options.timeout), None)
print "crash: " + str(crash)
print "timeout: " + str(timeout)
print "checksum: " + str(checksum)
print 'stdout: """'
print ''.join(output)
print '"""'
print 'stderr: """'
print ''.join(err)
print '"""'
print
if __name__ == '__main__':
optparser = optparse.OptionParser()
optparser.add_option('-p', '--port', action='store', default='mac',
'Platform to test (e.g., "mac", "chromium-mac", etc.')
optparser.add_option('-t', '--target', action='store', default='Release',
'build type ("Debug" or "Release")')
optparser.add_option('', '--timeout', action='store', default='2000',
'test timeout in milliseconds (2000 by default)')
optparser.add_option('', '--wrapper', action='store')
optparser.add_option('', '--no-pixel-tests', action='store_true',
default=False,
help='disable pixel-to-pixel PNG comparisons')
options, args = optparser.parse_args()
p = port.get(options.port, options)
run_tests(p, options, args)
print
if __name__ == '__main__':
optparser = optparse.OptionParser()
optparser.add_option(
'-p',
'--platform',
action='store',
default='mac',
help='Platform to test (e.g., "mac", "chromium-mac", etc.')
optparser.add_option('-t',
'--target',
action='store',
default='Release',
help='build type ("Debug" or "Release")')
optparser.add_option('',
'--timeout',
action='store',
default='2000',
help='test timeout in milliseconds (2000 by default)')
optparser.add_option('', '--wrapper', action='store')
optparser.add_option('',
'--no-pixel-tests',
action='store_true',
default=False,
help='disable pixel-to-pixel PNG comparisons')
options, args = optparser.parse_args()
p = port.get(options.platform, options)
run_tests(p, options, args)
def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
self._port = port.get('test', None)
self._exp = None
unittest.TestCase.__init__(self, testFunc)
def main(executive=Executive()):
"""Main function to produce new baselines."""
(options, target_options) = parse_options(sys.argv[1:])
# We need to create three different Port objects over the life of this
# script. |target_port_obj| is used to determine configuration information:
# location of the expectations file, names of ports to rebaseline, etc.
# |port_obj| is used for runtime functionality like actually diffing
# Then we create a rebaselining port to actual find and manage the
# baselines.
target_port_obj = port.get(None, target_options)
# Set up our logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format=('%(asctime)s %(filename)s:%(lineno)-3d '
'%(levelname)s %(message)s'),
datefmt='%y%m%d %H:%M:%S')
host_port_obj = get_host_port_object(options)
if not host_port_obj:
sys.exit(1)
# Verify 'platforms' option is valid.
if not options.platforms:
_log.error('Invalid "platforms" option. --platforms must be '
'specified in order to rebaseline.')
sys.exit(1)
platforms = [p.strip().lower() for p in options.platforms.split(',')]
for platform in platforms:
if not platform in REBASELINE_PLATFORM_ORDER:
_log.error('Invalid platform: "%s"' % (platform))
sys.exit(1)
# Adjust the platform order so rebaseline tool is running at the order of
# 'mac', 'win' and 'linux'. This is in same order with layout test baseline
# search paths. It simplifies how the rebaseline tool detects duplicate
# baselines. Check _IsDupBaseline method for details.
rebaseline_platforms = []
for platform in REBASELINE_PLATFORM_ORDER:
if platform in platforms:
rebaseline_platforms.append(platform)
options.html_directory = setup_html_directory(options.html_directory)
rebaselining_tests = set()
backup = options.backup
for platform in rebaseline_platforms:
rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
platform, options)
_log.info('')
log_dashed_string('Rebaseline started', platform)
if rebaseliner.run(backup):
# Only need to backup one original copy of test expectation file.
backup = False
log_dashed_string('Rebaseline done', platform)
else:
log_dashed_string('Rebaseline failed', platform, logging.ERROR)
rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
_log.info('')
log_dashed_string('Rebaselining result comparison started', None)
html_generator = HtmlGenerator(target_port_obj,
options,
rebaseline_platforms,
rebaselining_tests,
executive=executive)
html_generator.generate_html()
if not options.quiet:
html_generator.show_html()
log_dashed_string('Rebaselining result comparison done', None)
sys.exit(0)
