def CreatePageSet(self, options):
page_set_path = os.path.join(util.GetChromiumSrcDir(), 'tools', 'perf',
'page_sets')
ps = page_set_module.PageSet(archive_data_file='data/maps.json',
base_dir=page_set_path,
bucket=page_set_module.PUBLIC_BUCKET)
ps.AddUserStory(MapsPage(ps, ps.base_dir))
return ps
def testAbsoluteExpectations(self):
ps = page_set.PageSet()
page = page_module.Page('http://test.com/page5.html', ps)
page_org = page_module.Page('http://test.org/page5.html', ps)
page_https = page_module.Page('https://test.com/page5.html', ps)
self.assertExpectationEquals('fail', StubPlatform('win'), page)
self.assertExpectationEquals('pass', StubPlatform('win'), page_org)
self.assertExpectationEquals('pass', StubPlatform('win'), page_https)
def testGetUrlBaseDirAndFileForUrlBaseDir(self):
base_dir = os.path.dirname(__file__)
file_path = os.path.join(os.path.dirname(base_dir), 'otherdir',
'file.html')
ps = page_set.PageSet(base_dir=base_dir,
serving_dirs=[os.path.join('..', 'somedir', '')])
ps.AddUserStory(
page.Page('file://../otherdir/file.html', ps, ps.base_dir))
self.assertPathEqual(ps[0].file_path, file_path)
def testDisplayUrlForFileDirectories(self):
ps = page_set.PageSet(base_dir=os.path.dirname(__file__))
ps.AddUserStory(page.Page('file://../../otherdir/foo', ps,
ps.base_dir))
ps.AddUserStory(page.Page('file://../../otherdir/bar', ps,
ps.base_dir))
self.assertEquals(ps[0].display_name, 'foo')
self.assertEquals(ps[1].display_name, 'bar')
def testDisplayUrlForFileOfDifferentPaths(self):
ps = page_set.PageSet(base_dir=os.path.dirname(__file__))
ps.AddUserStory(
page.Page('file://../../somedir/foo.html', ps, ps.base_dir))
ps.AddUserStory(
page.Page('file://../../otherdir/bar.html', ps, ps.base_dir))
self.assertEquals(ps[0].display_name, 'somedir/foo.html')
self.assertEquals(ps[1].display_name, 'otherdir/bar.html')
def testDisplayUrlForFilesDifferingBySuffix(self):
ps = page_set.PageSet(base_dir=os.path.dirname(__file__))
ps.AddUserStory(
page.Page('file://../../otherdir/foo.html', ps, ps.base_dir))
ps.AddUserStory(
page.Page('file://../../otherdir/foo1.html', ps, ps.base_dir))
self.assertEquals(ps[0].display_name, 'foo.html')
self.assertEquals(ps[1].display_name, 'foo1.html')
def setUp(self):
ps = page_set.PageSet(file_path=os.path.dirname(__file__))
ps.AddUserStory(
page_module.Page('http://www.bar.com/', ps, ps.base_dir))
ps.AddUserStory(
page_module.Page('http://www.baz.com/', ps, ps.base_dir))
ps.AddUserStory(
page_module.Page('http://www.foo.com/', ps, ps.base_dir))
self.page_set = ps
def CreatePageSet(self, options):
ps = page_set.PageSet(
archive_data_file='../page_sets/data/octane.json',
file_path=os.path.abspath(__file__),
bucket=page_set.PUBLIC_BUCKET)
ps.AddUserStory(page_module.Page(
'http://octane-benchmark.googlecode.com/svn/latest/index.html?auto=1',
ps, ps.base_dir, make_javascript_deterministic=False))
return ps
def FinalizeImport(self):
self._model.FinalizeImport()
self._threads_to_records_map = (
tbm_module._GetRendererThreadsToInteractionRecordsMap(self._model))
self._ps = page_set.PageSet(file_path=os.path.dirname(__file__))
self._ps.AddUserStory(
page_module.Page('http://www.bar.com/', self._ps,
self._ps.base_dir))
self._results.WillRunPage(self._ps.pages[0])
def CreatePageSet(self, options):
ps = page_set.PageSet(
archive_data_file='../page_sets/data/kraken.json',
base_dir=os.path.dirname(os.path.abspath(__file__)),
bucket=page_set.PARTNER_BUCKET)
ps.AddUserStory(page_module.Page(
'http://krakenbenchmark.mozilla.org/kraken-1.1/driver.html',
ps, ps.base_dir))
return ps
def CreatePageSet(self, options):
page_set_path = os.path.join(util.GetChromiumSrcDir(), 'content',
'test', 'gpu', 'page_sets')
ps = page_set.PageSet(archive_data_file='data/maps.json',
make_javascript_deterministic=False,
file_path=page_set_path,
bucket=page_set.PUBLIC_BUCKET)
ps.AddPage(MapsPage(ps, ps.base_dir))
return ps
def CreatePageSet(self, options):
ps = page_set.PageSet(
archive_data_file='../page_sets/data/robohornet_pro.json',
# Measurement require use of real Date.now() for measurement.
make_javascript_deterministic=False,
file_path=os.path.abspath(__file__))
ps.AddPageWithDefaultRunNavigate(
'http://ie.microsoft.com/testdrive/performance/robohornetpro/')
return ps
def CreatePageSet(self, options):
page_set_path = os.path.join(
util.GetChromiumSrcDir(), 'tools', 'perf', 'page_sets')
ps = page_set_module.PageSet(
archive_data_file='data/maps.json',
make_javascript_deterministic=False,
file_path=page_set_path)
ps.AddPage(MapsPage(ps, ps.base_dir))
return ps
def CreatePageSet(self, options):
ps = page_set.PageSet(
archive_data_file='../page_sets/data/octane.json',
make_javascript_deterministic=False,
file_path=os.path.abspath(__file__))
ps.AddPageWithDefaultRunNavigate(
'http://octane-benchmark.googlecode.com/svn/latest/index.html?auto=1'
)
return ps
def testCloudBucket(self):
blank_ps = page_set.PageSet()
expected_bucket = None
self.assertEqual(blank_ps.bucket, expected_bucket)
public_ps = page_set.PageSet(bucket=page_set.PUBLIC_BUCKET)
expected_bucket = cloud_storage.PUBLIC_BUCKET
self.assertEqual(public_ps.bucket, expected_bucket)
partner_ps = page_set.PageSet(bucket=page_set.PARTNER_BUCKET)
expected_bucket = cloud_storage.PARTNER_BUCKET
self.assertEqual(partner_ps.bucket, expected_bucket)
internal_ps = page_set.PageSet(bucket=page_set.INTERNAL_BUCKET)
expected_bucket = cloud_storage.INTERNAL_BUCKET
self.assertEqual(internal_ps.bucket, expected_bucket)
self.assertRaises(ValueError, page_set.PageSet, bucket='garbage_bucket')
def CreatePageSet(self, options):
ps = page_set.PageSet(
archive_data_file='../page_sets/data/jetstream.json',
base_dir=os.path.dirname(os.path.abspath(__file__)),
bucket=page_set.INTERNAL_BUCKET)
ps.AddUserStory(page_module.Page(
'http://browserbench.org/JetStream/', ps, ps.base_dir,
make_javascript_deterministic=False))
return ps
def CreatePageSet(self, options):
dom_perf_dir = os.path.join(util.GetChromiumSrcDir(), 'data',
'dom_perf')
ps = page_set.PageSet(base_dir=dom_perf_dir)
for param in self.RUN_PARAMS:
ps.AddUserStory(
page_module.Page('file://run.html?reportInJS=1&run=%s' % param,
ps, ps.base_dir))
return ps
def TestUseLiveSitesFlag(self, expected_wpr_mode):
with tempfile.NamedTemporaryFile() as f:
run_state = shared_page_state.SharedPageState(
DummyTest(), self.options, page_set.PageSet())
fake_network_controller = FakeNetworkController()
run_state._PrepareWpr(fake_network_controller, f.name, None)
self.assertEquals(fake_network_controller.wpr_mode,
expected_wpr_mode)
self.assertEquals(fake_network_controller.archive_path, f.name)
def testRecordAndReplay(self):
test_archive = '/tmp/google.wpr'
google_url = 'http://www.google.com/'
foo_url = 'http://www.foo.com/'
archive_info_template = ("""
{
"archives": {
"%s": ["%s"]
}
}
""")
try:
ps = page_set.PageSet()
measurement = PageTestForReplay()
# First record an archive with only www.google.com.
self._options.browser_options.wpr_mode = wpr_modes.WPR_RECORD
# pylint: disable=protected-access
ps._wpr_archive_info = archive_info.WprArchiveInfo(
'', '', ps.bucket,
json.loads(archive_info_template % (test_archive, google_url)))
ps.pages = [page_module.Page(google_url, ps)]
all_results = self.RunMeasurement(measurement,
ps,
options=self._options)
self.assertEquals(0, len(all_results.failures))
# Now replay it and verify that google.com is found but foo.com is not.
self._options.browser_options.wpr_mode = wpr_modes.WPR_REPLAY
# pylint: disable=protected-access
ps._wpr_archive_info = archive_info.WprArchiveInfo(
'', '', ps.bucket,
json.loads(archive_info_template % (test_archive, foo_url)))
ps.pages = [page_module.Page(foo_url, ps)]
all_results = self.RunMeasurement(measurement,
ps,
options=self._options)
self.assertEquals(1, len(all_results.failures))
# pylint: disable=protected-access
ps._wpr_archive_info = archive_info.WprArchiveInfo(
'', '', ps.bucket,
json.loads(archive_info_template % (test_archive, google_url)))
ps.pages = [page_module.Page(google_url, ps)]
all_results = self.RunMeasurement(measurement,
ps,
options=self._options)
self.assertEquals(0, len(all_results.failures))
self.assertTrue(os.path.isfile(test_archive))
finally:
if os.path.isfile(test_archive):
os.remove(test_archive)
def setUp(self):
ps = page_set.PageSet()
self.p1 = page.Page(
'file://conformance/textures/tex-sub-image-2d.html', page_set=ps,
name='WebglConformance.conformance_textures_tex_sub_image_2d')
self.p2 = page.Page(
'file://othersuite/textures/tex-sub-image-3d.html', page_set=ps,
name='OtherSuite.textures_tex_sub_image_3d')
self.p3 = page.Page(
'file://othersuite/textures/tex-sub-image-3d.html', page_set=ps)
def CreatePageSet(self, options):
ps = page_set.PageSet(
archive_data_file='../page_sets/data/scirra.json',
make_javascript_deterministic=False,
file_path=os.path.abspath(__file__))
for url in ('http://www.scirra.com/labs/renderperf3/',
'http://www.scirra.com/demos/c2/renderperfgl/',
'http://www.scirra.com/demos/c2/renderperf2d/'):
ps.AddPageWithDefaultRunNavigate(url)
return ps
def CreatePageSet(self, options):
ps = page_set.PageSet(file_path=data_path,
user_agent_type='desktop',
serving_dirs=set(['']))
ps.AddPage(WebGLContextLostFromGPUProcessExitPage(ps, ps.base_dir))
ps.AddPage(
WebGLContextLostFromLoseContextExtensionPage(ps, ps.base_dir))
ps.AddPage(WebGLContextLostFromQuantityPage(ps, ps.base_dir))
ps.AddPage(WebGLContextLostFromSelectElementPage(ps, ps.base_dir))
return ps
def disabled_testRecordAndReplay(self):
# This test is disabled because it runs against live sites, and needs to be
# fixed. crbug.com/179038
test_archive = '/tmp/google.wpr'
google_url = 'http://www.google.com/'
foo_url = 'http://www.foo.com/'
archive_info_template = ("""
{
"archives": {
"%s": ["%s"]
}
}
""")
try:
ps = page_set.PageSet()
measurement = MeasurementForReplay()
# First record an archive with only www.google.com.
self._options.browser_options.wpr_mode = wpr_modes.WPR_RECORD
ps.wpr_archive_info = page_set_archive_info.PageSetArchiveInfo(
'', '',
json.loads(archive_info_template % (test_archive, google_url)))
ps.pages = [page_module.Page(google_url, ps)]
all_results = self.RunMeasurement(measurement,
ps,
options=self._options)
self.assertEquals(0, len(all_results.failures))
# Now replay it and verify that google.com is found but foo.com is not.
self._options.browser_options.wpr_mode = wpr_modes.WPR_REPLAY
ps.wpr_archive_info = page_set_archive_info.PageSetArchiveInfo(
'', '',
json.loads(archive_info_template % (test_archive, foo_url)))
ps.pages = [page_module.Page(foo_url, ps)]
all_results = self.RunMeasurement(measurement,
ps,
options=self._options)
self.assertEquals(1, len(all_results.failures))
ps.wpr_archive_info = page_set_archive_info.PageSetArchiveInfo(
'', '',
json.loads(archive_info_template % (test_archive, google_url)))
ps.pages = [page_module.Page(google_url, ps)]
all_results = self.RunMeasurement(measurement,
ps,
options=self._options)
self.assertEquals(0, len(all_results.failures))
self.assertTrue(os.path.isfile(test_archive))
finally:
if os.path.isfile(test_archive):
os.remove(test_archive)
def testMaxFailuresOptionIsRespected(self):
class TestPage(page_module.Page):
def __init__(self, *args, **kwargs):
super(TestPage, self).__init__(*args, **kwargs)
self.was_run = False
def RunNavigateSteps(self, action_runner):
self.was_run = True
raise Exception('Test exception')
class Test(page_test.PageTest):
def ValidatePage(self, *args):
pass
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
page1 = TestPage('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page1)
page2 = TestPage('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page2)
page3 = TestPage('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page3)
page4 = TestPage('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page4)
page5 = TestPage('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page5)
options = options_for_unittests.GetCopy()
options.output_format = 'none'
options.suppress_gtest_report = True
SetUpPageRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
page_runner.Run(Test(max_failures=2), ps, expectations, options,
results)
self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
# Runs up to max_failures+1 failing tests before stopping, since
# every tests after max_failures failures have been encountered
# may all be passing.
self.assertEquals(3, len(results.failures))
self.assertTrue(page1.was_run)
self.assertTrue(page2.was_run)
self.assertTrue(page3.was_run)
self.assertFalse(page4.was_run)
self.assertFalse(page5.was_run)
def CreatePageSet(self, options):
"""Makes a PageSet for Dromaeo benchmarks."""
# Subclasses are expected to define a class member called query_param.
if not hasattr(self, 'query_param'):
raise NotImplementedError('query_param not in Dromaeo benchmark.')
url = 'file://index.html?%s&automated' % self.query_param
dromaeo_dir = os.path.join(util.GetChromiumSrcDir(), 'chrome', 'test',
'data', 'dromaeo')
ps = page_set.PageSet(file_path=dromaeo_dir)
ps.AddPageWithDefaultRunNavigate(url)
return ps
def testDiscardFirstResult(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
ps.pages.append(
page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
ps.pages.append(
page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
class Measurement(page_measurement.PageMeasurement):
@property
def discard_first_result(self):
return True
def MeasurePage(self, *args):
pass
options = options_for_unittests.GetCopy()
options.output_format = 'none'
options.reset_results = None
options.upload_results = None
options.results_label = None
options.page_repeat = 1
options.pageset_repeat = 1
SetUpPageRunnerArguments(options)
results = page_runner.Run(Measurement(), ps, expectations, options)
self.assertEquals(0, len(results.successes))
self.assertEquals(0, len(results.failures))
options.page_repeat = 1
options.pageset_repeat = 2
SetUpPageRunnerArguments(options)
results = page_runner.Run(Measurement(), ps, expectations, options)
self.assertEquals(2, len(results.successes))
self.assertEquals(0, len(results.failures))
options.page_repeat = 2
options.pageset_repeat = 1
SetUpPageRunnerArguments(options)
results = page_runner.Run(Measurement(), ps, expectations, options)
self.assertEquals(2, len(results.successes))
self.assertEquals(0, len(results.failures))
options.output_format = 'html'
options.page_repeat = 1
options.pageset_repeat = 1
SetUpPageRunnerArguments(options)
results = page_runner.Run(Measurement(), ps, expectations, options)
self.assertEquals(0, len(results.successes))
self.assertEquals(0, len(results.failures))
def CreatePageSet(self, options):
ps = page_set.PageSet(
archive_data_file='../page_sets/data/sunspider.json',
file_path=os.path.abspath(__file__),
bucket=page_set.PARTNER_BUCKET)
ps.AddUserStory(
page_module.Page(_URL,
ps,
ps.base_dir,
make_javascript_deterministic=False))
return ps
def CreatePageSet(self, options):
ps = page_set.PageSet(base_dir=data_path, serving_dirs=set(['']))
ps.AddUserStory(GPUProcessCrashesExactlyOnce(ps, ps.base_dir))
ps.AddUserStory(WebGLContextLostFromGPUProcessExitPage(
ps, ps.base_dir))
ps.AddUserStory(
WebGLContextLostFromLoseContextExtensionPage(ps, ps.base_dir))
ps.AddUserStory(WebGLContextLostFromQuantityPage(ps, ps.base_dir))
ps.AddUserStory(WebGLContextLostFromSelectElementPage(ps, ps.base_dir))
ps.AddUserStory(WebGLContextLostInHiddenTabPage(ps, ps.base_dir))
return ps
def CreatePageSet(self, options):
ps = page_set.PageSet(
base_dir=os.path.dirname(os.path.abspath(__file__)),
archive_data_file='../page_sets/data/speedometer.json',
bucket=page_set.PUBLIC_BUCKET)
ps.AddUserStory(
page_module.Page('http://browserbench.org/Speedometer/',
ps,
ps.base_dir,
make_javascript_deterministic=False))
return ps
def CreatePageSet(self, options):
features = ['WebGL', 'Canvas']
ps = page_set.PageSet(
description='Tests GPU acceleration is reported as active',
user_agent_type='desktop',
file_path='')
for feature in features:
ps.AddPage(ChromeGpuPage(page_set=ps, feature=feature))
return ps
