def test_get_raptor_test_list_override_page_timeout(create_args):
args = create_args(test="amazon", page_timeout=9999, browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
assert test_list[0]["page_timeout"] == 9999
def main(args=sys.argv[1:]):
args = parse_args()
commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
LOG = get_default_logger(component='raptor-main')
LOG.info("received command line arguments: %s" % str(args))
# if a test name specified on command line, and it exists, just run that one
# otherwise run all available raptor tests that are found for this browser
raptor_test_list = get_raptor_test_list(args)
# ensure we have at least one valid test to run
if len(raptor_test_list) == 0:
LOG.critical("abort: no tests found")
sys.exit(1)
LOG.info("raptor tests scheduled to run:")
for next_test in raptor_test_list:
LOG.info(next_test['name'])
raptor = Raptor(args.app, args.binary, args.run_local, args.obj_path)
raptor.start_control_server()
for next_test in raptor_test_list:
raptor.run_test(next_test)
success = raptor.process_results()
raptor.clean_up()
if not success:
# didn't get test results; test timed out or crashed, etc. we want job to fail
LOG.critical("error: no raptor test results were found")
os.sys.exit(1)
def test_get_raptor_test_list_chrome(create_args):
args = create_args(app="chrome",
test="raptor-speedometer")
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-speedometer-chrome'
def main(args=sys.argv[1:]):
args = parse_args()
commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
LOG = get_default_logger(component='raptor-main')
# if a test name specified on command line, and it exists, just run that one
# otherwise run all available raptor tests that are found for this browser
raptor_test_list = get_raptor_test_list(args)
# ensure we have at least one valid test to run
if len(raptor_test_list) == 0:
LOG.critical("abort: no tests found")
sys.exit(1)
LOG.info("raptor tests scheduled to run:")
for next_test in raptor_test_list:
LOG.info(next_test['name'])
raptor = Raptor(args.app, args.binary)
raptor.start_control_server()
for next_test in raptor_test_list:
raptor.run_test(next_test)
raptor.process_results()
raptor.clean_up()
def test_get_raptor_test_list_geckoview(create_args):
args = create_args(app="geckoview",
test="raptor-unity-webgl")
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-unity-webgl-geckoview'
def test_get_raptor_test_list_override_page_timeout(create_args):
args = create_args(test="raptor-tp6-google-firefox", page_timeout=9999)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
assert test_list[0]['page_timeout'] == 9999
def test_get_raptor_test_list_using_live_sites(create_args):
args = create_args(test="amazon", live_sites=True, browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
assert test_list[0]["use_live_sites"] == "true"
assert test_list[0]["playback"] is None
def test_get_raptor_test_list_refbrow(create_args):
args = create_args(app="refbrow",
test="raptor-speedometer",
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-speedometer-refbrow'
def test_get_raptor_test_list_fenix(create_args):
args = create_args(app="fenix",
test="raptor-speedometer",
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
# we don't have any actual fenix tests yet
assert len(test_list) == 0
def test_get_raptor_test_list_chrome(create_args):
args = create_args(app="chrome",
test="raptor-speedometer",
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "raptor-speedometer-chrome"
def test_get_raptor_test_list_gecko_profiling(create_args):
args = create_args(test="raptor-tp6-google-firefox", gecko_profile=True)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
assert test_list[0]['gecko_profile'] is True
assert test_list[0]['page_cycles'] == 3
def test_get_raptor_test_list_debug_mode(create_args):
args = create_args(test="raptor-tp6-google-firefox", debug_mode=True)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
assert test_list[0]['debug_mode'] is True
assert test_list[0]['page_cycles'] == 2
def test_get_raptor_test_list_add_test_url_params(create_args):
args = create_args(test="amazon", test_url_params="c=4", browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
query_params = parse_qs(urlsplit(test_list[0]["test_url"]).query)
assert query_params.get("c") == ["4"]
def test_get_raptor_test_list_debug_mode(create_args):
args = create_args(test="amazon", debug_mode=True, browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
assert test_list[0]["debug_mode"] is True
assert test_list[0]["page_cycles"] == 2
def test_get_raptor_test_list_override_page_cycles(create_args):
args = create_args(test="raptor-tp6-google-firefox",
page_cycles=99,
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "raptor-tp6-google-firefox"
assert test_list[0]["page_cycles"] == 99
def test_get_raptor_test_list_add_test_url_params(create_args):
args = create_args(test="raptor-tp6-google-firefox",
test_url_params='c=4',
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
query_params = parse_qs(urlsplit(test_list[0]['test_url']).query)
assert query_params.get('c') == ['4']
def test_get_raptor_test_list_using_live_sites(create_args):
args = create_args(test="raptor-tp6-amazon-firefox",
live_sites=True,
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-amazon-firefox'
assert test_list[0]['use_live_sites'] == 'true'
assert test_list[0]['playback'] is None
def test_get_raptor_test_list_gecko_profiling_enabled(create_args):
args = create_args(test="amazon", gecko_profile=True, browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
assert test_list[0]["gecko_profile"] is True
assert test_list[0].get("gecko_profile_entries") == "14000000"
assert test_list[0].get("gecko_profile_interval") == "1"
assert test_list[0].get("gecko_profile_threads") is None
def test_get_raptor_test_list_firefox(create_args):
args = create_args()
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 4
subtests = ['raptor-tp6-google-firefox', 'raptor-tp6-amazon-firefox',
'raptor-tp6-facebook-firefox', 'raptor-tp6-youtube-firefox']
for next_subtest in test_list:
assert next_subtest['name'] in subtests
def test_get_raptor_test_list_gecko_profiling_enabled(create_args):
args = create_args(test="raptor-tp6-google-firefox",
gecko_profile=True,
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
assert test_list[0]['gecko_profile'] is True
assert test_list[0].get('gecko_profile_entries') == '14000000'
assert test_list[0].get('gecko_profile_interval') == '1'
assert test_list[0].get('gecko_profile_threads') is None
def main(args=sys.argv[1:]):
args = parse_args()
commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
LOG = get_default_logger(component='raptor-main')
LOG.info("raptor-start")
LOG.info("received command line arguments: %s" % str(args))
# if a test name specified on command line, and it exists, just run that one
# otherwise run all available raptor tests that are found for this browser
raptor_test_list = get_raptor_test_list(args, mozinfo.os)
# ensure we have at least one valid test to run
if len(raptor_test_list) == 0:
LOG.critical("abort: no tests found")
sys.exit(1)
LOG.info("raptor tests scheduled to run:")
for next_test in raptor_test_list:
LOG.info(next_test['name'])
raptor = Raptor(args.app, args.binary, args.run_local, args.obj_path,
args.gecko_profile, args.gecko_profile_interval,
args.gecko_profile_entries, args.symbols_path)
raptor.start_control_server()
for next_test in raptor_test_list:
if 'page_timeout' not in next_test.keys():
next_test['page_timeout'] = 120000
if 'page_cycles' not in next_test.keys():
next_test['page_cycles'] = 1
raptor.run_test(next_test, timeout=int(next_test['page_timeout']))
success = raptor.process_results()
raptor.clean_up()
if not success:
# didn't get test results; test timed out or crashed, etc. we want job to fail
LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found")
os.sys.exit(1)
# if we have results but one test page timed out (i.e. one tp6 test page didn't load
# but others did) we still dumped PERFHERDER_DATA for the successfull pages but we
# want the overall test job to marked as a failure
pages_that_timed_out = raptor.get_page_timeout_list()
if len(pages_that_timed_out) > 0:
for _page in pages_that_timed_out:
LOG.critical(
"TEST-UNEXPECTED-FAIL: test '%s' timed out loading test page: %s"
% (_page['test_name'], _page['url']))
os.sys.exit(1)
def test_get_raptor_test_list_gecko_profiling_disabled(create_args):
args = create_args(test="raptor-tp6-google-firefox",
gecko_profile=False,
gecko_profile_entries=42,
gecko_profile_interval=100,
gecko_profile_threads=['Foo'],
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
assert test_list[0].get('gecko_profile') is None
assert test_list[0].get('gecko_profile_entries') is None
assert test_list[0].get('gecko_profile_interval') is None
assert test_list[0].get('gecko_profile_threads') is None
def test_get_raptor_test_list_gecko_profiling_enabled_args_override(create_args):
args = create_args(test="raptor-tp6-google-firefox",
gecko_profile=True,
gecko_profile_entries=42,
gecko_profile_interval=100,
gecko_profile_threads=['Foo'],
browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
assert test_list[0]['gecko_profile'] is True
assert test_list[0]['gecko_profile_entries'] == '42'
assert test_list[0]['gecko_profile_interval'] == '100'
assert test_list[0]['gecko_profile_threads'] == 'Foo'
def test_get_raptor_test_list_firefox(create_args):
args = create_args(browser_cycles=1)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 4
subtests = [
"raptor-tp6-unittest-google-firefox",
"raptor-tp6-unittest-amazon-firefox",
"raptor-tp6-unittest-facebook-firefox",
"raptor-tp6-unittest-youtube-firefox",
]
for next_subtest in test_list:
assert next_subtest["name"] in subtests
def test_get_raptor_test_list_gecko_profiling_disabled(create_args):
args = create_args(
test="raptor-tp6-google-firefox",
gecko_profile=False,
gecko_profile_entries=42,
gecko_profile_interval=100,
gecko_profile_threads=["Foo"],
browser_cycles=1,
)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "raptor-tp6-google-firefox"
assert test_list[0].get("gecko_profile") is None
assert test_list[0].get("gecko_profile_entries") is None
assert test_list[0].get("gecko_profile_interval") is None
assert test_list[0].get("gecko_profile_threads") is None
def test_get_raptor_test_list_gecko_profiling_enabled_args_override(
create_args):
args = create_args(
test="raptor-tp6-google-firefox",
gecko_profile=True,
gecko_profile_entries=42,
gecko_profile_interval=100,
gecko_profile_threads=["Foo"],
browser_cycles=1,
)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "raptor-tp6-google-firefox"
assert test_list[0]["gecko_profile"] is True
assert test_list[0]["gecko_profile_entries"] == "42"
assert test_list[0]["gecko_profile_interval"] == "100"
assert test_list[0]["gecko_profile_threads"] == "Foo"
def test_get_raptor_test_list_gecko_profiling_enabled_extra_args_override(
create_args):
args = create_args(
test="amazon",
gecko_profile=True,
gecko_profile_entries=42,
gecko_profile_interval=100,
gecko_profile_extra_threads=["Foo", "Oof"],
gecko_profile_threads="String,Rope",
browser_cycles=1,
)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
assert test_list[0]["gecko_profile"] is True
assert test_list[0]["gecko_profile_entries"] == "42"
assert test_list[0]["gecko_profile_interval"] == "100"
assert test_list[0]["gecko_profile_threads"] == "String,Rope,Foo,Oof"
def test_get_raptor_test_list_gecko_profiling_enabled_args_override(
create_args):
args = create_args(
test="amazon",
gecko_profile=True,
gecko_profile_entries=42,
gecko_profile_interval=100,
gecko_profile_threads="Foo",
gecko_profile_features="Mood,UserNetWorth",
browser_cycles=1,
)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
assert test_list[0]["gecko_profile"] is True
assert test_list[0]["gecko_profile_entries"] == "42"
assert test_list[0]["gecko_profile_interval"] == "100"
assert test_list[0]["gecko_profile_threads"] == "Foo"
assert test_list[0]["gecko_profile_features"] == "Mood,UserNetWorth"
def test_get_raptor_test_list_gecko_profiling_disabled_args_override(
create_args):
args = create_args(
test="amazon",
gecko_profile=False,
gecko_profile_entries=42,
gecko_profile_interval=100,
gecko_profile_threads=["Foo"],
gecko_profile_features=["Temperature"],
browser_cycles=1,
)
test_list = get_raptor_test_list(args, mozinfo.os)
assert len(test_list) == 1
assert test_list[0]["name"] == "amazon"
assert test_list[0].get("gecko_profile") is None
assert test_list[0].get("gecko_profile_entries") is None
assert test_list[0].get("gecko_profile_interval") is None
assert test_list[0].get("gecko_profile_threads") is None
assert test_list[0].get("gecko_profile_features") is None
def main(args=sys.argv[1:]):
args = parse_args()
args.extra_prefs = parse_preferences(args.extra_prefs or [])
if args.enable_fission:
args.extra_prefs.update({
"fission.autostart": True,
"dom.serviceWorkers.parent_intercept": True,
"browser.tabs.documentchannel": True,
})
if args.extra_prefs and args.extra_prefs.get("fission.autostart", False):
args.enable_fission = True
commandline.setup_logging("raptor", args, {"tbpl": sys.stdout})
LOG.info("Python version: %s" % sys.version)
LOG.info("raptor-start")
if args.debug_mode:
LOG.info("debug-mode enabled")
LOG.info("received command line arguments: %s" % str(args))
# if a test name specified on command line, and it exists, just run that one
# otherwise run all available raptor tests that are found for this browser
raptor_test_list = get_raptor_test_list(args, mozinfo.os)
raptor_test_names = [
raptor_test["name"] for raptor_test in raptor_test_list
]
# ensure we have at least one valid test to run
if len(raptor_test_list) == 0:
LOG.critical("test '{}' could not be found for {}".format(
args.test, args.app))
sys.exit(1)
LOG.info("raptor tests scheduled to run:")
for next_test in raptor_test_list:
LOG.info(next_test["name"])
if not args.browsertime:
if args.app == "firefox":
raptor_class = WebExtensionFirefox
elif args.app in CHROMIUM_DISTROS:
raptor_class = WebExtensionDesktopChrome
else:
raptor_class = WebExtensionAndroid
else:
def raptor_class(*inner_args, **inner_kwargs):
outer_kwargs = vars(args)
# peel off arguments that are specific to browsertime
for key in outer_kwargs.keys():
if key.startswith("browsertime_"):
value = outer_kwargs.pop(key)
inner_kwargs[key] = value
if args.app == "firefox" or args.app in CHROMIUM_DISTROS:
klass = BrowsertimeDesktop
else:
klass = BrowsertimeAndroid
return klass(*inner_args, **inner_kwargs)
try:
raptor = raptor_class(
args.app,
args.binary,
run_local=args.run_local,
noinstall=args.noinstall,
installerpath=args.installerpath,
obj_path=args.obj_path,
gecko_profile=args.gecko_profile,
gecko_profile_interval=args.gecko_profile_interval,
gecko_profile_entries=args.gecko_profile_entries,
symbols_path=args.symbols_path,
host=args.host,
power_test=args.power_test,
cpu_test=args.cpu_test,
memory_test=args.memory_test,
live_sites=args.live_sites,
cold=args.cold,
is_release_build=args.is_release_build,
debug_mode=args.debug_mode,
post_startup_delay=args.post_startup_delay,
activity=args.activity,
intent=args.intent,
interrupt_handler=SignalHandler(),
enable_webrender=args.enable_webrender,
extra_prefs=args.extra_prefs or {},
device_name=args.device_name,
no_conditioned_profile=args.no_conditioned_profile,
disable_perf_tuning=args.disable_perf_tuning,
conditioned_profile_scenario=args.conditioned_profile_scenario,
chimera=args.chimera,
project=args.project,
verbose=args.verbose)
except Exception:
traceback.print_exc()
LOG.critical(
"TEST-UNEXPECTED-FAIL: could not initialize the raptor test runner"
)
os.sys.exit(1)
success = raptor.run_tests(raptor_test_list, raptor_test_names)
if not success:
# if we have results but one test page timed out (i.e. one tp6 test page didn't load
# but others did) we still dumped PERFHERDER_DATA for the successfull pages but we
# want the overall test job to marked as a failure
pages_that_timed_out = raptor.get_page_timeout_list()
if pages_that_timed_out:
for _page in pages_that_timed_out:
message = [
("TEST-UNEXPECTED-FAIL", "test '%s'" % _page["test_name"]),
("timed out loading test page",
"waiting for pending metrics"),
]
if _page.get("pending_metrics") is not None:
LOG.warning("page cycle {} has pending metrics: {}".format(
_page["page_cycle"], _page["pending_metrics"]))
LOG.critical(" ".join("%s: %s" % (subject, msg)
for subject, msg in message))
else:
# we want the job to fail when we didn't get any test results
# (due to test timeout/crash/etc.)
LOG.critical(
"TEST-UNEXPECTED-FAIL: no raptor test results were found for %s"
% ", ".join(raptor_test_names))
os.sys.exit(1)
# if we're running browsertime in the CI, we want to zip the result dir
if args.browsertime and not args.run_local:
result_dir = raptor.results_handler.result_dir()
if os.path.exists(result_dir):
LOG.info("Creating tarball at %s" % result_dir + ".tgz")
with tarfile.open(result_dir + ".tgz", "w:gz") as tar:
tar.add(result_dir, arcname=os.path.basename(result_dir))
LOG.info("Removing %s" % result_dir)
shutil.rmtree(result_dir)
# when running raptor locally with gecko profiling on, use the view-gecko-profile
# tool to automatically load the latest gecko profile in profiler.firefox.com
if args.gecko_profile and args.run_local:
if os.environ.get("DISABLE_PROFILE_LAUNCH", "0") == "1":
LOG.info(
"Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1"
)
else:
view_gecko_profile_from_raptor()
