def view_gecko_profile(ffox_bin): # automatically load the latest talos gecko-profile archive in profiler.firefox.com LOG_GECKO = RaptorLogger(component='raptor-view-gecko-profile') if sys.platform.startswith('win') and not ffox_bin.endswith(".exe"): ffox_bin = ffox_bin + ".exe" if not os.path.exists(ffox_bin): LOG_GECKO.info( "unable to find Firefox bin, cannot launch view-gecko-profile") return profile_zip = os.environ.get('RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE', None) if profile_zip is None or not os.path.exists(profile_zip): LOG_GECKO.info("No local talos gecko profiles were found so not " "launching profiler.firefox.com") return # need the view-gecko-profile tool, it's in repo/testing/tools repo_dir = os.environ.get('MOZ_DEVELOPER_REPO_DIR', None) if repo_dir is None: LOG_GECKO.info( "unable to find MOZ_DEVELOPER_REPO_DIR, can't launch view-gecko-profile" ) return view_gp = os.path.join(repo_dir, 'testing', 'tools', 'view_gecko_profile', 'view_gecko_profile.py') if not os.path.exists(view_gp): LOG_GECKO.info( "unable to find the view-gecko-profile tool, cannot launch it") return command = ['python', view_gp, '-b', ffox_bin, '-p', profile_zip] LOG_GECKO.info('Auto-loading this profile in perfhtml.io: %s' % profile_zip) LOG_GECKO.info(command) # if the view-gecko-profile tool fails to launch for some reason, we don't # want to crash talos! just dump error and finsh up talos as usual try: view_profile = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # that will leave it running in own instance and let talos finish up except Exception as e: LOG_GECKO.info( "failed to launch view-gecko-profile tool, exeption: %s" % e) return time.sleep(5) ret = view_profile.poll() if ret is None: LOG_GECKO.info("view-gecko-profile successfully started as pid %d" % view_profile.pid) else: LOG_GECKO.error( 'view-gecko-profile process failed to start, poll returned: %s' % ret)
def view_gecko_profile_from_raptor(): # automatically load the latest raptor gecko-profile archive in profiler.firefox.com LOG_GECKO = RaptorLogger(component='raptor-view-gecko-profile') profile_zip_path = os.environ.get('RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE', None) if profile_zip_path is None or not os.path.exists(profile_zip_path): LOG_GECKO.info("No local raptor gecko profiles were found so not " "launching profiler.firefox.com") return LOG_GECKO.info("Profile saved locally to: %s" % profile_zip_path) view_gecko_profile(profile_zip_path)
""" from __future__ import absolute_import import json import os import tempfile import zipfile import fnmatch import mozfile from logger.logger import RaptorLogger from mozgeckoprofiler import ProfileSymbolicator, save_gecko_profile here = os.path.dirname(os.path.realpath(__file__)) LOG = RaptorLogger(component="raptor-gecko-profile") class GeckoProfile(object): """ Handle Gecko profiling. This allow to collect Gecko profiling data and to zip results in one file. """ def __init__(self, upload_dir, raptor_config, test_config): self.upload_dir = upload_dir self.raptor_config, self.test_config = raptor_config, test_config self.cleanup = True # Create a temporary directory into which the tests can put # their profiles. These files will be assembled into one big
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os import shutil import socket from logger.logger import RaptorLogger from wptserve import server, handlers LOG = RaptorLogger(component="raptor-benchmark") here = os.path.abspath(os.path.dirname(__file__)) class Benchmark(object): """utility class for running benchmarks in raptor""" def __init__(self, config, test): self.config = config self.test = test # bench_dir is where we will download all mitmproxy required files # when running locally it comes from obj_path via mozharness/mach if self.config.get("obj_path", None) is not None: self.bench_dir = self.config.get("obj_path") else: # in production it is ../tasks/task_N/build/tests/raptor/raptor/... # 'here' is that path, we can start with that self.bench_dir = here
webext_dir = os.path.join(here, "..", "webext") paths.append(webext_dir) for path in paths: if not os.path.exists(path): raise IOError("%s does not exist. " % path) sys.path.insert(0, path) from cmdline import FIREFOX_ANDROID_APPS from condprof.client import get_profile, ProfileNotFoundError from condprof.util import get_current_platform from logger.logger import RaptorLogger from gecko_profile import GeckoProfile from results import RaptorResultsHandler LOG = RaptorLogger(component="raptor-perftest") # - mozproxy.utils LOG displayed INFO messages even when LOG.error() was used in mitm.py mpu.LOG = RaptorLogger(component="raptor-mitmproxy") try: from mozbuild.base import MozbuildObject build = MozbuildObject.from_environment(cwd=here) except ImportError: build = None POST_DELAY_CONDPROF = 1000 POST_DELAY_DEBUG = 3000 POST_DELAY_DEFAULT = 30000
import json import os import re from six.moves.urllib.parse import parse_qs, urlsplit, urlunsplit, urlencode, unquote from logger.logger import RaptorLogger from manifestparser import TestManifest from utils import bool_from_str, transform_platform, transform_subtest from constants.raptor_tests_constants import YOUTUBE_PLAYBACK_MEASURE here = os.path.abspath(os.path.dirname(__file__)) raptor_ini = os.path.join(here, 'raptor.ini') tests_dir = os.path.join(here, 'tests') LOG = RaptorLogger(component='raptor-manifest') LIVE_SITE_TIMEOUT_MULTIPLIER = 1.2 required_settings = [ 'alert_threshold', 'apps', 'lower_is_better', 'measure', 'page_cycles', 'test_url', 'scenario_time', 'type', 'unit', ]
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os import re from logger.logger import RaptorLogger LOG = RaptorLogger(component='raptor-power') def init_android_power_test(raptor): upload_dir = os.getenv("MOZ_UPLOAD_DIR") if not upload_dir: LOG.critical("% power test ignored; MOZ_UPLOAD_DIR unset" % raptor.config["app"]) return # Set the screen-off timeout to two (2) hours, since the device will be running # disconnected, and would otherwise turn off the screen, thereby halting # execution of the test. Save the current value so we can restore it later # since it is a persistent change. raptor.screen_off_timeout = raptor.device.shell_output( "settings get system screen_off_timeout").strip() raptor.device.shell_output( "settings put system screen_off_timeout 7200000") # Set the screen brightness to ~50% for consistency of measurements across # devices and save its current value to restore it later. Screen brightness
from mozlog import commandline from mozprofile.cli import parse_preferences from browsertime import BrowsertimeDesktop, BrowsertimeAndroid from cmdline import parse_args, CHROMIUM_DISTROS from logger.logger import RaptorLogger from manifest import get_raptor_test_list from signal_handler import SignalHandler from utils import view_gecko_profile_from_raptor from webextension import ( WebExtensionFirefox, WebExtensionDesktopChrome, WebExtensionAndroid, ) LOG = RaptorLogger(component="raptor-main") def main(args=sys.argv[1:]): args = parse_args() args.extra_prefs = parse_preferences(args.extra_prefs or []) if args.enable_fission: args.extra_prefs.update({ "fission.autostart": True, "dom.serviceWorkers.parent_intercept": True, "browser.tabs.documentchannel": True, }) if args.extra_prefs and args.extra_prefs.get("fission.autostart", False):
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os import shutil import socket from logger.logger import RaptorLogger from wptserve import server, handlers LOG = RaptorLogger(component='raptor-benchmark') here = os.path.abspath(os.path.dirname(__file__)) class Benchmark(object): """utility class for running benchmarks in raptor""" def __init__(self, config, test): self.config = config self.test = test # bench_dir is where we will download all mitmproxy required files # when running locally it comes from obj_path via mozharness/mach if self.config.get("obj_path", None) is not None: self.bench_dir = self.config.get("obj_path") else: # in production it is ../tasks/task_N/build/tests/raptor/raptor/... # 'here' is that path, we can start with that self.bench_dir = here
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import time import threading from logger.logger import RaptorLogger LOG = RaptorLogger(component='raptor-cpu') class AndroidCPUProfiler(object): """AndroidCPUProfiler is used to measure CPU usage over time for an app in testing. Polls usage at a defined interval and then submits this as a supporting perfherder data measurement. ``` # Initialize the profiler and start polling for CPU usage of the app cpu_profiler = AndroidCPUProfiler(raptor, poll_interval=1) cpu_profiler.start_polling() # Or call the following to perform the commands above cpu_profiler = start_android_cpu_profiler(raptor) # Run test... # Stop measuring and generate perfherder data cpu_profiler.generate_android_cpu_profile("raptor-tp6m-1") ```
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # class to process, format, and report raptor test results # received from the raptor control server from __future__ import absolute_import import json import os from logger.logger import RaptorLogger from output import Output LOG = RaptorLogger(component='raptor-results-handler') class RaptorResultsHandler(): """Handle Raptor test results""" def __init__(self, config=None): self.config = config self.results = [] self.page_timeout_list = [] self.images = [] self.supporting_data = None def add(self, new_result_json): # add to results LOG.info("received results in RaptorResultsHandler.add") new_result = RaptorTestResult(new_result_json) self.results.append(new_result)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, division import os import re import time from logger.logger import RaptorLogger from mozdevice import ADBError, ADBTimeoutError LOG = RaptorLogger(component="raptor-power") P2_PATH = "/sys/class/power_supply/battery/input_suspend" G5_PATH = "/sys/class/power_supply/battery/charging_enabled" S7_PATH = "/sys/class/power_supply/battery/batt_slate_mode" def get_device_type(device, timeout=10): """Returns the type of device being tested. Currently it can either be Pixel 2, Moto G5, or Samsung S7.""" device_type = device.shell_output("getprop ro.product.model", timeout=timeout) if device_type == "Pixel 2": pass elif device_type == "Moto G (5)": pass elif device_type == "SM-G930F": # samsung s7 galaxy (exynos)
"""Utility functions for Raptor""" # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os import yaml from distutils.util import strtobool from logger.logger import RaptorLogger from mozgeckoprofiler import view_gecko_profile LOG = RaptorLogger(component="raptor-utils") here = os.path.dirname(os.path.realpath(__file__)) external_tools_path = os.environ.get("EXTERNALTOOLSPATH", None) if external_tools_path is not None: # running in production via mozharness TOOLTOOL_PATH = os.path.join(external_tools_path, "tooltool.py") def transform_platform(str_to_transform, config_platform, config_processor=None): """Transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest' transforms to 'mitmproxy-rel-bin-osx.manifest'. Also transform '{x64}' if needed for 64 bit / win 10""" if "{platform}" not in str_to_transform and "{x64}" not in str_to_transform:
"""Utility functions for Raptor""" # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os import subprocess import sys import time import yaml from logger.logger import RaptorLogger LOG = RaptorLogger(component='raptor-utils') here = os.path.dirname(os.path.realpath(__file__)) external_tools_path = os.environ.get('EXTERNALTOOLSPATH', None) if external_tools_path is not None: # running in production via mozharness TOOLTOOL_PATH = os.path.join(external_tools_path, 'tooltool.py') def transform_platform(str_to_transform, config_platform, config_processor=None): """Transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest' transforms to 'mitmproxy-rel-bin-osx.manifest'. Also transform '{x64}' if needed for 64 bit / win 10"""
# License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os from mozdevice import ADBDevice from logger.logger import RaptorLogger from performance_tuning import tune_performance from perftest import PerftestAndroid from .base import Browsertime LOG = RaptorLogger(component="raptor-browsertime-android") class BrowsertimeAndroid(PerftestAndroid, Browsertime): """Android setup and configuration for browsertime When running raptor-browsertime tests on android, we create the profile (and set the proxy prefs in the profile that is using playback) but we don't need to copy it onto the device because geckodriver takes care of that. We tell browsertime to use our profile (we pass it in with the firefox.profileTemplate arg); browsertime creates a copy of that and passes that into geckodriver. Geckodriver then takes the profile and copies it onto the mobile device's sdcard for us; and then it even writes the geckoview app config.yaml file onto the device, which points the app to the profile on the sdcard. Therefore, raptor doesn't have to copy the profile onto the scard (and create the config.yaml) file ourselves. Also note when using playback, the nss certificate db is created as usual when
#!/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import from logger.logger import RaptorLogger from perftest import PerftestDesktop from .base import Browsertime LOG = RaptorLogger(component="raptor-browsertime-desktop") class BrowsertimeDesktop(PerftestDesktop, Browsertime): def __init__(self, *args, **kwargs): super(BrowsertimeDesktop, self).__init__(*args, **kwargs) @property def browsertime_args(self): binary_path = self.config["binary"] LOG.info("binary_path: {}".format(binary_path)) args_list = ["--viewPort", "1024x768"] if self.config["app"] in ( "chrome", "chromium", ):
webext_dir = os.path.join(here, "..", "webext") paths.append(webext_dir) for path in paths: if not os.path.exists(path): raise IOError("%s does not exist. " % path) sys.path.insert(0, path) from cmdline import FIREFOX_ANDROID_APPS from condprof.client import get_profile, ProfileNotFoundError from condprof.util import get_current_platform from logger.logger import RaptorLogger from gecko_profile import GeckoProfile from results import RaptorResultsHandler LOG = RaptorLogger(component="raptor-perftest") try: from mozbuild.base import MozbuildObject build = MozbuildObject.from_environment(cwd=here) except ImportError: build = None class Perftest(object): """Abstract base class for perftests that execute via a subharness, either Raptor or browsertime.""" __metaclass__ = ABCMeta
import tempfile import time import mozcrash from cpu import start_android_cpu_profiler from logger.logger import RaptorLogger from mozdevice import ADBDevice from performance_tuning import tune_performance from perftest import PerftestAndroid from power import (init_android_power_test, finish_android_power_test, enable_charging, disable_charging) from signal_handler import SignalHandlerException from utils import write_yml_file from webextension.base import WebExtension LOG = RaptorLogger(component="raptor-webext-android") class WebExtensionAndroid(PerftestAndroid, WebExtension): def __init__(self, app, binary, activity=None, intent=None, **kwargs): super(WebExtensionAndroid, self).__init__(app, binary, profile_class="firefox", **kwargs) self.config.update({"activity": activity, "intent": intent}) self.remote_test_root = os.path.abspath( os.path.join(os.sep, "sdcard", "raptor")) self.remote_profile = os.path.join(self.remote_test_root, "profile") self.os_baseline_data = None
# file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os import shutil from mozpower import MozPower from mozrunner import runners from logger.logger import RaptorLogger from outputhandler import OutputHandler from perftest import PerftestDesktop from .base import WebExtension LOG = RaptorLogger(component="raptor-webext-desktop") class WebExtensionDesktop(PerftestDesktop, WebExtension): def __init__(self, *args, **kwargs): super(WebExtensionDesktop, self).__init__(*args, **kwargs) # create the desktop browser runner LOG.info("creating browser runner using mozrunner") self.output_handler = OutputHandler(verbose=self.config["verbose"]) process_args = {"processOutputLine": [self.output_handler]} firefox_args = ["--allow-downgrade"] runner_cls = runners[self.config["app"]] self.runner = runner_cls( self.config["binary"], profile=self.profile,
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # originally from talos_process.py from __future__ import absolute_import import json from logger.logger import RaptorLogger LOG = RaptorLogger(component="raptor-output-handler") class OutputHandler(object): def __init__(self, verbose=False): self.proc = None self.verbose = verbose def __call__(self, line): if not line.strip(): return line = line.decode("utf-8", errors="replace") try: data = json.loads(line) except ValueError: self.process_output(line) return if isinstance(data, dict) and "action" in data:
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import os from logger.logger import RaptorLogger here = os.path.abspath(os.path.dirname(__file__)) webext_dir = os.path.join(os.path.dirname(here), "webext", "raptor") LOG = RaptorLogger(component="raptor-gen-test-config") FILE_CONTENTS = """// this file is auto-generated by raptor, do not edit directly function getTestConfig() {{ return {{ "cs_port": "{control_server_port}", "test_name": "{test}", "test_settings_url": "http://{host}:{control_server_port}/json/{test}.json", "post_startup_delay": "{post_startup_delay}", "benchmark_port": "{benchmark_port}", "host": "{host}", "debug_mode": "{debug_mode}", "browser_cycle": "{browser_cycle}" }}; }} """ def gen_test_config(
# control server for raptor performance framework # communicates with the raptor browser webextension from __future__ import absolute_import import BaseHTTPServer import datetime import json import os import shutil import socket import threading import time from logger.logger import RaptorLogger LOG = RaptorLogger(component="raptor-control-server") here = os.path.abspath(os.path.dirname(__file__)) def MakeCustomHandlerClass(results_handler, shutdown_browser, handle_gecko_profile, background_app, foreground_app): class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler, object): """ Control server expects messages of the form {'type': 'messagetype', 'data':...} Each message is given a key which is calculated as If type is 'webext_status', then
# class to process, format, and report raptor test results # received from the raptor control server from __future__ import absolute_import import json import os import shutil from abc import ABCMeta, abstractmethod from io import open import six from logger.logger import RaptorLogger from output import RaptorOutput, BrowsertimeOutput LOG = RaptorLogger(component="perftest-results-handler") KNOWN_TEST_MODIFIERS = [ "condprof-settled", "fission", "live", "gecko-profile", "cold", "webrender", ] @six.add_metaclass(ABCMeta) class PerftestResultsHandler(object): """Abstract base class to handle perftest results""" def __init__(self, gecko_profile=False,
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # some parts of this originally taken from /testing/talos/talos/output.py """output raptor test results""" from __future__ import absolute_import import filters import json import os from logger.logger import RaptorLogger LOG = RaptorLogger(component='raptor-output') class Output(object): """class for raptor output""" def __init__(self, results, supporting_data, subtest_alert_on): """ - results : list of RaptorTestResult instances """ self.results = results self.summarized_results = {} self.supporting_data = supporting_data self.summarized_supporting_data = [] self.summarized_screenshots = [] self.subtest_alert_on = subtest_alert_on
import json import os import requests import time from benchmark import Benchmark from cmdline import CHROMIUM_DISTROS from control_server import RaptorControlServer from gen_test_config import gen_test_config from logger.logger import RaptorLogger from memory import generate_android_memory_profile from perftest import Perftest from results import RaptorResultsHandler LOG = RaptorLogger(component="raptor-webext") here = os.path.abspath(os.path.dirname(__file__)) webext_dir = os.path.join(here, "..", "..", "webext") class WebExtension(Perftest): """Container class for WebExtension.""" def __init__(self, *args, **kwargs): self.raptor_webext = None self.control_server = None self.cpu_profiler = None super(WebExtension, self).__init__(*args, **kwargs)
import json import os import re from six.moves.urllib.parse import parse_qs, urlsplit, urlunsplit, urlencode, unquote from logger.logger import RaptorLogger from manifestparser import TestManifest from utils import bool_from_str, transform_platform, transform_subtest from constants.raptor_tests_constants import YOUTUBE_PLAYBACK_MEASURE here = os.path.abspath(os.path.dirname(__file__)) raptor_ini = os.path.join(here, "raptor.ini") tests_dir = os.path.join(here, "tests") LOG = RaptorLogger(component="raptor-manifest") LIVE_SITE_TIMEOUT_MULTIPLIER = 1.2 required_settings = [ "alert_threshold", "apps", "lower_is_better", "measure", "page_cycles", "test_url", "scenario_time", "type", "unit", ]
from __future__ import absolute_import from abc import ABCMeta, abstractmethod import os import json import re import six import mozprocess from benchmark import Benchmark from logger.logger import RaptorLogger from perftest import Perftest from results import BrowsertimeResultsHandler LOG = RaptorLogger(component="raptor-browsertime") DEFAULT_CHROMEVERSION = "77" BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT = 120 # 2 minutes BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT = None # Disable output timeout for benchmark tests class Browsertime(Perftest): """Abstract base class for Browsertime""" __metaclass__ = ABCMeta @property @abstractmethod def browsertime_args(self): pass
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import time import threading from logger.logger import RaptorLogger LOG = RaptorLogger(component="raptor-cpu") class AndroidCPUProfiler(object): """AndroidCPUProfiler is used to measure CPU usage over time for an app in testing. Polls usage at a defined interval and then submits this as a supporting perfherder data measurement. ``` # Initialize the profiler and start polling for CPU usage of the app cpu_profiler = AndroidCPUProfiler(raptor, poll_interval=1) cpu_profiler.start_polling() # Or call the following to perform the commands above cpu_profiler = start_android_cpu_profiler(raptor) # Run test... # Stop measuring and generate perfherder data cpu_profiler.generate_android_cpu_profile("browsertime-tp6m") ```