def Run(self, args):
if args.json_output_file:
possible_browser = browser_finder.FindBrowser(args)
if args.browser_type in (
'exact', 'release', 'release_x64', 'debug', 'debug_x64', 'canary'):
args.browser_type = 'reference'
possible_reference_browser = browser_finder.FindBrowser(args)
else:
possible_reference_browser = None
with open(args.json_output_file, 'w') as f:
f.write(_GetJsonTestList(possible_browser, possible_reference_browser,
args.tests, args.num_shards))
else:
_PrintTestList(args.tests)
return 0
def testOneTab(self):
ps = page_set.PageSet()
page = page_module.Page(
'file:///' +
os.path.join('..', '..', 'unittest_data', 'blank.html'),
ps,
base_dir=os.path.dirname(__file__))
ps.pages.append(page)
class TestOneTab(page_test.PageTest):
def __init__(self,
test_method_name,
action_name_to_run='',
needs_browser_restart_after_each_run=False):
super(TestOneTab,
self).__init__(test_method_name, action_name_to_run,
needs_browser_restart_after_each_run)
self._browser = None
def SetUpBrowser(self, browser):
self._browser = browser
self._browser.tabs.New()
def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
assert len(self._browser.tabs) == 1
test = TestOneTab('RunTest')
with page_runner.PageRunner(ps) as runner:
options = options_for_unittests.GetCopy()
possible_browser = browser_finder.FindBrowser(options)
results = page_test.PageTestResults()
runner.Run(options, possible_browser, test, results)
def testUserAgent(self):
ps = page_set.PageSet()
page = page_module.Page(
'file:///' +
os.path.join('..', '..', 'unittest_data', 'blank.html'),
ps,
base_dir=os.path.dirname(__file__))
ps.pages.append(page)
ps.user_agent_type = 'tablet'
class TestUserAgent(page_test.PageTest):
def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
actual_user_agent = tab.EvaluateJavaScript(
'window.navigator.userAgent')
expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
assert actual_user_agent.strip() == expected_user_agent
# This is so we can check later that the test actually made it into this
# function. Previously it was timing out before even getting here, which
# should fail, but since it skipped all the asserts, it slipped by.
self.hasRun = True # pylint: disable=W0201
test = TestUserAgent('RunTest')
with page_runner.PageRunner(ps) as runner:
options = options_for_unittests.GetCopy()
possible_browser = browser_finder.FindBrowser(options)
results = page_test.PageTestResults()
runner.Run(options, possible_browser, test, results)
self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def runCredentialsTest(
self, # pylint: disable=R0201
credentials_backend,
results):
ps = page_set.PageSet()
page = page_module.Page('http://www.google.com', ps)
page.credentials = "test"
ps.pages.append(page)
did_run = [False]
with tempfile.NamedTemporaryFile() as f:
f.write(SIMPLE_CREDENTIALS_STRING)
f.flush()
ps.credentials_path = f.name
class TestThatInstallsCredentialsBackend(page_test.PageTest):
def __init__(self, credentials_backend):
super(TestThatInstallsCredentialsBackend,
self).__init__('RunTest')
self._credentials_backend = credentials_backend
def SetUpBrowser(self, browser):
browser.credentials.AddBackend(self._credentials_backend)
def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
did_run[0] = True
test = TestThatInstallsCredentialsBackend(credentials_backend)
with page_runner.PageRunner(ps) as runner:
options = options_for_unittests.GetCopy()
possible_browser = browser_finder.FindBrowser(options)
runner.Run(options, possible_browser, test, results)
return did_run[0]
def testRunPageWithStartupUrl(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
expectations = test_expectations.TestExpectations()
page = page_module.Page(
'file://blank.html', ps, base_dir=util.GetUnittestDataDir(),
startup_url='about:blank')
ps.pages.append(page)
class Measurement(page_test.PageTest):
def __init__(self):
super(Measurement, self).__init__()
self.browser_restarted = False
def CustomizeBrowserOptionsForSinglePage(self, ps, options):
self.browser_restarted = True
super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps,
options)
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.page_repeat = 2
options.output_formats = ['none']
options.suppress_gtest_report = True
if not browser_finder.FindBrowser(options):
return
test = Measurement()
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(test, ps, expectations, options, results)
self.assertEquals('about:blank', options.browser_options.startup_url)
self.assertTrue(test.browser_restarted)
def ProcessCommandLineArgs(cls, parser, args):
if not args.positional_args:
possible_browser = (browser_finder.FindBrowser(args)
if args.browser_type else None)
_PrintBenchmarkList(_Benchmarks(), possible_browser)
sys.exit(-1)
input_benchmark_name = args.positional_args[0]
matching_benchmarks = _MatchBenchmarkName(input_benchmark_name)
if not matching_benchmarks:
print >> sys.stderr, 'No benchmark named "%s".' % input_benchmark_name
print >> sys.stderr
_PrintBenchmarkList(_Benchmarks(), None)
sys.exit(-1)
if len(matching_benchmarks) > 1:
print >> sys.stderr, ('Multiple benchmarks named "%s".' %
input_benchmark_name)
print >> sys.stderr, 'Did you mean one of these?'
print >> sys.stderr
_PrintBenchmarkList(matching_benchmarks, None)
sys.exit(-1)
benchmark_class = matching_benchmarks.pop()
if len(args.positional_args) > 1:
parser.error('Too many arguments.')
assert issubclass(
benchmark_class,
benchmark.Benchmark), ('Trying to run a non-Benchmark?!')
benchmark.ProcessCommandLineArgs(parser, args)
benchmark_class.ProcessCommandLineArgs(parser, args)
cls._benchmark = benchmark_class
def _CreateBrowser(self,
autotest_ext=False,
auto_login=True,
gaia_login=False,
username=None,
password=None):
"""Finds and creates a browser for tests. if autotest_ext is True,
also loads the autotest extension"""
options = options_for_unittests.GetCopy()
if autotest_ext:
extension_path = os.path.join(util.GetUnittestDataDir(),
'autotest_ext')
assert os.path.isdir(extension_path)
self._load_extension = extension_to_load.ExtensionToLoad(
path=extension_path,
browser_type=options.browser_type,
is_component=True)
options.extensions_to_load = [self._load_extension]
browser_to_create = browser_finder.FindBrowser(options)
self.assertTrue(browser_to_create)
browser_options = options.browser_options
browser_options.create_browser_with_oobe = True
browser_options.auto_login = auto_login
browser_options.gaia_login = gaia_login
if username is not None:
browser_options.username = username
if password is not None:
browser_options.password = password
return browser_to_create.Create(options)
def Main(args, start_dir, top_level_dir, runner=None):
"""Unit test suite that collects all test cases for telemetry."""
# Add unittest_data to the path so we can import packages from it.
util.AddDirToPythonPath(util.GetUnittestDataDir())
default_options = browser_options.BrowserFinderOptions()
default_options.browser_type = 'any'
parser = default_options.CreateParser('run_tests [options] [test names]')
parser.add_option('--repeat-count',
dest='run_test_repeat_count',
type='int',
default=1,
help='Repeats each a provided number of times.')
parser.add_option('-d',
'--also-run-disabled-tests',
dest='run_disabled_tests',
action='store_true',
default=False,
help='Ignore @Disabled and @Enabled restrictions.')
_, args = parser.parse_args(args)
if default_options.verbosity == 0:
logging.getLogger().setLevel(logging.WARN)
from telemetry.core import browser_finder
try:
browser_to_create = browser_finder.FindBrowser(default_options)
except browser_finder.BrowserFinderException, ex:
logging.error(str(ex))
return 1
def Main(args, start_dir, top_level_dir, runner=None):
"""Unit test suite that collects all test cases for telemetry."""
# Add unittest_data to the path so we can import packages from it.
unittest_data_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'unittest_data'))
sys.path.append(unittest_data_dir)
default_options = browser_options.BrowserOptions()
default_options.browser_type = 'any'
parser = default_options.CreateParser('run_tests [options] [test names]')
parser.add_option('--repeat-count',
dest='run_test_repeat_count',
type='int',
default=1,
help='Repeats each a provided number of times.')
parser.add_option('-d',
'--also-run-disabled-tests',
dest='run_disabled_tests',
action='store_true',
default=False,
help='Also run tests decorated with @DisabledTest.')
_, args = parser.parse_args(args)
logging_level = logging.getLogger().getEffectiveLevel()
if default_options.verbosity == 0:
logging.getLogger().setLevel(logging.WARN)
from telemetry.core import browser_finder
try:
browser_to_create = browser_finder.FindBrowser(default_options)
except browser_finder.BrowserFinderException, ex:
logging.error(str(ex))
return 1
def Main(args):
options = browser_options.BrowserFinderOptions()
parser = options.CreateParser('telemetry_perf_test.py')
options, args = parser.parse_args(args)
browser_to_create = browser_finder.FindBrowser(options)
assert browser_to_create
with browser_to_create.Create(options) as b:
tab = b.tabs[0]
# Measure round-trip-time for evaluate
times = []
for i in range(1000):
start = time.time()
tab.EvaluateJavaScript('%i * 2' % i)
times.append(time.time() - start)
N = float(len(times))
avg = sum(times, 0.0) / N
squared_diffs = [(t - avg) * (t - avg) for t in times]
stdev = sum(squared_diffs, 0.0) / (N - 1)
times.sort()
percentile_75 = times[int(0.75 * N)]
print "%s: avg=%f; stdev=%f; min=%f; 75th percentile = %f" % (
"Round trip time (seconds)", avg, stdev, min(times), percentile_75)
return 0
def testBasicHostingAndRangeRequests(self):
options = options_for_unittests.GetCopy()
browser_to_create = browser_finder.FindBrowser(options)
with browser_to_create.Create() as b:
b.Start()
b.SetHTTPServerDirectories(util.GetUnittestDataDir())
t = b.tabs[0]
t.Navigate(b.http_server.UrlOf('/blank.html'))
t.WaitForDocumentReadyStateToBeComplete()
x = t.EvaluateJavaScript('document.body.innerHTML')
x = x.strip()
# Test basic html hosting.
self.assertEquals(x, 'Hello world')
file_size = self.test_file_size
last_byte = file_size - 1
# Test byte range request: no end byte.
self.CheckContentHeaders(b, t, '0-', '0-%d' % last_byte, file_size)
# Test byte range request: greater than zero start byte.
self.CheckContentHeaders(b, t, '100-', '100-%d' % last_byte,
file_size - 100)
# Test byte range request: explicit byte range.
self.CheckContentHeaders(b, t, '2-500', '2-500', '499')
# Test byte range request: no start byte.
self.CheckContentHeaders(b, t, '-228',
'%d-%d' % (file_size - 228, last_byte),
'228')
# Test byte range request: end byte less than start byte.
self.CheckContentHeaders(b, t, '100-5', '100-%d' % last_byte,
file_size - 100)
def Run(self, options):
"""Kick off the process.
Args:
options: Instance of BrowserFinderOptions to search for proper browser.
Returns:
The path of generated profile or existing profile if --profile-dir
is given. Could be None if it's generated on default location
(e.g., crytohome on CrOS).
"""
possible_browser = browser_finder.FindBrowser(options)
is_cros = possible_browser.browser_type.startswith('cros')
out_dir = None
if is_cros:
self.Create(options, None)
else:
# Decide profile output path.
out_dir = (options.browser_options.profile_dir or
os.path.abspath(os.path.join(
tempfile.gettempdir(), self._profile_name, self._profile_name)))
if not os.path.exists(out_dir):
self.Create(options, out_dir)
return out_dir
def CreateBrowser(self, extra_browser_args=None, profile_type=None):
assert not self._browser
options = options_for_unittests.GetCopy()
if profile_type:
# TODO(jeremy): crbug.com/243912 profiles are only implemented on
# Desktop.
is_running_on_desktop = not (
options.browser_type.startswith('android')
or options.browser_type.startswith('cros'))
if not is_running_on_desktop:
logging.warn("Desktop-only test, skipping.")
return None
options.profile_type = profile_type
if extra_browser_args:
options.extra_browser_args.extend(extra_browser_args)
browser_to_create = browser_finder.FindBrowser(options)
if not browser_to_create:
raise Exception('No browser found, cannot continue test.')
self._browser = browser_to_create.Create()
unittest_data_dir = os.path.join(os.path.dirname(__file__), '..', '..',
'unittest_data')
self._browser.SetHTTPServerDirectories(unittest_data_dir)
return self._browser
def testLoginStatus(self):
extension_path = os.path.join(os.path.dirname(__file__),
'autotest_ext')
load_extension = extension_to_load.ExtensionToLoad(
extension_path, True)
options = options_for_unittests.GetCopy()
options.extensions_to_load = [load_extension]
browser_to_create = browser_finder.FindBrowser(options)
self.assertTrue(browser_to_create)
with browser_to_create.Create() as b:
extension = b.extensions[load_extension]
self.assertTrue(extension)
extension.ExecuteJavaScript('''
chrome.autotestPrivate.loginStatus(function(s) {
window.__autotest_result = s;
});
''')
login_status = extension.EvaluateJavaScript(
'window.__autotest_result')
self.assertEquals(type(login_status), dict)
self.assertEquals(not self._is_guest,
login_status['isRegularUser'])
self.assertEquals(self._is_guest, login_status['isGuest'])
self.assertEquals(login_status['email'], self._email)
self.assertFalse(login_status['isScreenLocked'])
def Run(self, args):
possible_browser = browser_finder.FindBrowser(args)
test_suite, result = self.RunOneSuite(possible_browser, args)
results = [result]
failed_tests = json_results.FailedTestNames(test_suite, result)
retry_limit = args.retry_limit
while retry_limit and failed_tests:
args.positional_args = failed_tests
args.exact_test_filter = True
_, result = self.RunOneSuite(possible_browser, args)
results.append(result)
failed_tests = json_results.FailedTestNames(test_suite, result)
retry_limit -= 1
full_results = json_results.FullResults(args, test_suite, results)
json_results.WriteFullResultsIfNecessary(args, full_results)
err_occurred, err_str = json_results.UploadFullResultsIfNecessary(
args, full_results)
if err_occurred:
for line in err_str.splitlines():
logging.error(line)
return 1
return json_results.ExitCodeFromFullResults(full_results)
def testRunPageWithStartupUrl(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
expectations = test_expectations.TestExpectations()
page = page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
page.startup_url = 'about:blank'
ps.pages.append(page)
class Measurement(page_measurement.PageMeasurement):
def __init__(self):
super(Measurement, self).__init__()
self.browser_restarted = False
def CustomizeBrowserOptionsForSinglePage(self, ps, options):
self.browser_restarted = True
super(Measurement,
self).CustomizeBrowserOptionsForSinglePage(ps, options)
def MeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.page_repeat = 2
options.output_format = 'none'
if not browser_finder.FindBrowser(options):
return
test = Measurement()
SetUpPageRunnerArguments(options)
page_runner.Run(test, ps, expectations, options)
self.assertEquals('about:blank', options.browser_options.startup_url)
self.assertTrue(test.browser_restarted)
def testTabCallByReference(self):
options = options_for_unittests.GetCopy()
browser_to_create = browser_finder.FindBrowser(options)
with browser_to_create.Create() as b:
tab = b.tabs[0]
tab.Navigate('http://www.google.com/')
b.tabs[0].WaitForDocumentReadyStateToBeInteractiveOrBetter()
def testVersionDetection(self):
options = options_for_unittests.GetCopy()
browser_to_create = browser_finder.FindBrowser(options)
with browser_to_create.Create() as b:
# pylint: disable=W0212
self.assertTrue(b._browser_backend._inspector_protocol_version > 0)
self.assertTrue(b._browser_backend._chrome_branch_number > 0)
self.assertTrue(b._browser_backend._webkit_base_revision > 0)
def setUpClass(cls):
cls._number_of_tabs = 4
cls._profile_dir = _GenerateBrowserProfile(cls._number_of_tabs)
cls._options = options_for_unittests.GetCopy()
cls._options.browser_options.AppendExtraBrowserArgs(
['--restore-last-session'])
cls._options.browser_options.profile_dir = cls._profile_dir
cls._browser_to_create = browser_finder.FindBrowser(cls._options)
def Main(benchmark_dir):
benchmarks = discover.DiscoverClasses(benchmark_dir,
os.path.join(benchmark_dir, '..'),
page_benchmark.PageBenchmark)
options = browser_options.BrowserOptions()
parser = options.CreateParser('%prog <page_set>')
page_runner.PageRunner.AddCommandLineOptions(parser)
recorder = RecordPage(benchmarks)
recorder.AddCommandLineOptions(parser)
_, args = parser.parse_args()
if len(args) != 1:
parser.print_usage()
sys.exit(1)
ps = page_set.PageSet.FromFile(args[0])
# Set the archive path to something temporary.
temp_target_wpr_file_path = tempfile.mkstemp()[1]
ps.wpr_archive_info.AddNewTemporaryRecording(temp_target_wpr_file_path)
# Do the actual recording.
options.wpr_mode = wpr_modes.WPR_RECORD
recorder.CustomizeBrowserOptions(options)
possible_browser = browser_finder.FindBrowser(options)
if not possible_browser:
print >> sys.stderr, """No browser found.\n
Use --browser=list to figure out which are available.\n"""
sys.exit(1)
results = page_test.PageTestResults()
with page_runner.PageRunner(ps) as runner:
runner.Run(options, possible_browser, recorder, results)
if results.page_failures:
logging.warning(
'Some pages failed. The recording has not been updated for '
'these pages.')
logging.warning(
'Failed pages: %s', '\n'.join(
[failure['page'].url for failure in results.page_failures]))
if results.skipped_pages:
logging.warning('Some pages were skipped. The recording has not been '
'updated for these pages.')
logging.warning(
'Skipped pages: %s', '\n'.join(
[skipped['page'].url for skipped in results.skipped_pages]))
if results.page_successes:
# Update the metadata for the pages which were recorded.
ps.wpr_archive_info.AddRecordedPages(
[page['page'] for page in results.page_successes])
else:
os.remove(temp_target_wpr_file_path)
return min(255, len(results.page_failures))
def testCloseReferencedTab(self):
options = options_for_unittests.GetCopy()
browser_to_create = browser_finder.FindBrowser(options)
with browser_to_create.Create() as b:
b.tabs.New()
tab = b.tabs[0]
tab.Navigate('http://www.google.com/')
tab.Close()
self.assertEquals(1, len(b.tabs))
def Run(self, args):
if args.json_output_file:
possible_browser = browser_finder.FindBrowser(args)
with open(args.json_output_file, 'w') as f:
f.write(
_GetJsonTestList(possible_browser, args.tests,
args.num_shards))
else:
_PrintTestList(args.tests)
return 0
def testBrowserCreation(self):
options = options_for_unittests.GetCopy()
browser_to_create = browser_finder.FindBrowser(options)
if not browser_to_create:
raise Exception('No browser found, cannot continue test.')
with browser_to_create.Create() as b:
self.assertEquals(1, len(b.tabs))
# Different browsers boot up to different things
assert b.tabs[0].url
def ProcessCommandLineArgs(cls, parser, args):
# We retry failures by default unless we're running a list of tests
# explicitly.
if not args.retry_limit and not args.positional_args:
args.retry_limit = 3
try:
possible_browser = browser_finder.FindBrowser(args)
except browser_finder_exceptions.BrowserFinderException, ex:
parser.error(ex)
def Run(test, page_set, expectations, finder_options, results):
"""Runs a given test against a given page_set with the given options."""
test.ValidatePageSet(page_set)
# Create a possible_browser with the given options.
try:
possible_browser = browser_finder.FindBrowser(finder_options)
except browser_finder.BrowserTypeRequiredException, e:
sys.stderr.write(str(e) + '\n')
sys.exit(-1)
def testExtensionNotLoaded(self):
"""Querying an extension that was not loaded will return None"""
extension_path = os.path.join(util.GetUnittestDataDir(), 'simple_extension')
options = options_for_unittests.GetCopy()
load_extension = extension_to_load.ExtensionToLoad(
extension_path, options.browser_type)
browser_to_create = browser_finder.FindBrowser(options)
with browser_to_create.Create() as b:
if b.supports_extensions:
self.assertRaises(KeyError, lambda: b.extensions[load_extension])
def CustomizeBrowserOptions(self, options):
options.AppendExtraBrowserArgs([
'--disable-gesture-requirement-for-media-playback',
'--disable-domain-blocking-for-3d-apis',
'--disable-gpu-process-crash-limit', '--enable-unsafe-es3-apis'
])
target_os = browser_finder.FindBrowser(
options.finder_options).target_os
if target_os.startswith('win'):
options.AppendExtraBrowserArgs(['--use-gl=desktop'])
def RunTestOnPageSet(self, test, ps, results):
test.CustomizeBrowserOptions(self._options)
possible_browser = browser_finder.FindBrowser(self._options)
if not possible_browser:
self.PrintParseError(
'No browser found.\n'
'Use --browser=list to figure out which are available.')
with page_runner.PageRunner(ps) as runner:
runner.Run(self._options, possible_browser, test, results)
def testCryptohomeMounted(self):
options = options_for_unittests.GetCopy()
browser_to_create = browser_finder.FindBrowser(options)
if not browser_to_create:
raise Exception('No browser found, cannot continue test.')
with browser_to_create.Create() as b:
self.assertEquals(1, len(b.tabs))
self.assertTrue(b.tabs[0].url)
self.assertTrue(self._IsCryptohomeMounted())
self.assertFalse(self._IsCryptohomeMounted())
def _GetVariationsBrowserArgs(self, finder_options):
variations_dir = os.path.join(os.path.dirname(__file__), os.pardir,
os.pardir, os.pardir, 'testing',
'variations')
target_os = browser_finder.FindBrowser(finder_options).target_os
base_variations_path = os.path.join(variations_dir,
'fieldtrial_testing_config.json')
return fieldtrial_util.GenerateArgs(
base_variations_path,
os.path.join(
variations_dir, 'fieldtrial_testing_config_%s.json' %
self._FixupTargetOS(target_os)))