def __init__(self, app, binary, run_local=False, obj_path=None,
gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
symbols_path=None):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.config['gecko_profile'] = gecko_profile
self.config['gecko_profile_interval'] = gecko_profile_interval
self.config['gecko_profile_entries'] = gecko_profile_entries
self.config['symbols_path'] = symbols_path
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.gecko_profiler = None
self.post_startup_delay = 30000 # raptor webext pause time after browser startup
# Create the profile; for geckoview we want a firefox profile type
if self.config['app'] == 'geckoview':
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] == "geckoview":
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBAndroid(verbose=True)
self.device.clear_logcat()
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(
binary, profile=self.profile, process_args=process_args)
self.log.info("raptor config: %s" % str(self.config))
def build_browser_profile(self):
if (self.config["app"] in ["chrome", "chromium", "chrome-m"]
or self.config.get("conditioned_profile") is None):
self.profile = create_profile(self.profile_class)
else:
# use mozprofile to create a profile for us, from our conditioned profile's path
self.profile = create_profile(
self.profile_class, profile=self.get_conditioned_profile())
# Merge extra profile data from testing/profiles
with open(os.path.join(self.profile_data_dir, "profiles.json"),
"r") as fh:
base_profiles = json.load(fh)["raptor"]
for profile in base_profiles:
path = os.path.join(self.profile_data_dir, profile)
LOG.info("Merging profile: {}".format(path))
self.profile.merge(path)
if self.config["extra_prefs"].get("fission.autostart", False):
LOG.info("Enabling fission via browser preferences")
LOG.info("Browser preferences: {}".format(
self.config["extra_prefs"]))
self.profile.set_preferences(self.config["extra_prefs"])
# share the profile dir with the config and the control server
self.config["local_profile_dir"] = self.profile.profile
LOG.info("Local browser profile: {}".format(self.profile.profile))
def test_create_profile(tmpdir, app, cls):
path = tmpdir.strpath
if cls is None:
with pytest.raises(NotImplementedError):
create_profile(app)
return
profile = create_profile(app, profile=path)
assert isinstance(profile, BaseProfile)
assert profile.__class__ == cls
assert profile.profile == path
def run(self, metadata):
if self.get_arg("directory") is not None:
# no need to create one or load a conditioned one
return
# XXX we'll use conditioned profiles later
profile = create_profile(app="firefox")
# mozprofile.Profile.__del__ silently deletes the profile
# it creates in a non-deterministic time (garbage collected) by
# calling cleanup. We override this silly behavior here.
profile.cleanup = self._cleanup
prefs = metadata.get_options("browser_prefs")
if prefs == {}:
prefs["mozperftest"] = "true"
# apply custom user prefs if any
user_js = self.get_arg("user-js")
if user_js is not None:
self.info("Applying use prefs from %s" % user_js)
default_prefs = dict(Preferences.read_prefs(user_js))
prefs.update(default_prefs)
profile.set_preferences(prefs)
self.info("Created profile at %s" % profile.profile)
self._created_dirs.append(profile.profile)
self.set_arg("profile-directory", profile.profile)
return metadata
def __init__(self, app, binary):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor')
self.control_server = None
self.playback = None
# Create the profile
pref_file = os.path.join(here, 'preferences',
'{}.json'.format(self.config['app']))
prefs = {}
if os.path.isfile(pref_file):
with open(pref_file, 'r') as fh:
prefs = json.load(fh)
try:
self.profile = create_profile(self.config['app'],
preferences=prefs)
except NotImplementedError:
self.profile = None
# Create the runner
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
def __init__(self, app, binary):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor')
self.control_server = None
self.playback = None
# Create the profile
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# Create the runner
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(
binary, profile=self.profile, process_args=process_args)
def get_profile(self, metadata):
# XXX we'll use conditioned profiles
from mozprofile import create_profile
profile = create_profile(app="firefox")
prefs = metadata["browser"]["prefs"]
profile.set_preferences(prefs)
self.info("Created profile at %s" % profile.profile)
self._created_dirs.append(profile.profile)
return profile
def create_browser_profile(self):
self.profile = create_profile(self.profile_class)
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# add profile dir to our config
self.config['local_profile_dir'] = self.profile.profile
def __init__(self, recording):
self.config = {}
self.config['host'] = '127.0.0.1'
self.config['profile'] = create_profile('firefox')
self.config['binary'] = "org.mozilla.geckoview_example"
self.config['app'] = "geckoview"
self.config['local_profile_dir'] = self.config['profile'].profile
self.config['recording'] = os.path.join(os.getcwd(), recording)
self.log = get_default_logger(component='recorder-main - ')
self.proxy_process = None
self.device = None
def fresh_profile(profile, customization_data):
from mozprofile import create_profile # NOQA
# XXX on android we mgiht need to run it on the device?
LOG("Creating a fresh profile")
new_profile = create_profile(app="firefox")
prefs = customization_data["prefs"]
prefs.update(_PREFS)
new_profile.set_preferences(prefs)
extensions = []
for name, url in customization_data["addons"].items():
LOG("Downloading addon %s" % name)
extension = download_file(url)
extensions.append(extension)
new_profile.addons.install(extensions)
shutil.copytree(new_profile.profile, profile)
return profile
def cli(record, certutil, url, path):
# create profile
profile = create_profile("firefox")
print("Created profile: {}".format(profile.profile))
mitmproxy_home = os.path.join(os.path.expanduser("~"), ".mitmproxy")
cert = os.path.join(mitmproxy_home, "mitmproxy-ca-cert.cer")
# start mitmdump
scripts = os.path.join(os.getcwd(), "scripts")
mitmdump = os.path.join(os.getcwd(), "utils", "mitmdump")
if record:
command = [mitmdump, "--wfile", path]
else:
command = [
mitmdump,
"--replay-kill-extra",
"--script",
" ".join(
[os.path.join(scripts, "alternate-server-replay.py"), path]),
]
try:
print(command)
mitmdump_process = subprocess.Popen(command)
# create certificate database
certdb = "sql:{}/".format(profile.profile)
print("Creating certificate database")
command = [certutil, "-N", "-v", "-d", certdb, "--empty-password"]
subprocess.call(command)
# install mitmproxy certificate
command = [
certutil,
"-A",
"-d",
certdb,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
cert,
]
print("Installing {} into certificate database".format(cert))
subprocess.call(command)
# verify certificate is installed
command = [certutil, "-d", certdb, "-L"]
assert "mitmproxy-cert" in subprocess.check_output(command)
# setup device
device = ADBAndroid()
device.create_socket_connection("reverse", "tcp:8080", "tcp:8080")
device_storage = "/sdcard/raptor"
device_profile = os.path.join(device_storage, "profile")
if device.is_dir(device_storage):
device.rm(device_storage, recursive=True)
device.mkdir(device_storage)
device.mkdir(device_profile)
userjs = os.path.join(profile.profile, "user.js")
with open(userjs) as f:
prefs = f.readlines()
prefs = [p for p in prefs if "network.proxy" not in p]
with open(userjs, "w") as f:
f.writelines(prefs)
proxy = {
"network.proxy.type": 1,
"network.proxy.http": "127.0.0.1",
"network.proxy.http_port": 8080,
"network.proxy.ssl": "127.0.0.1",
"network.proxy.ssl_port": 8080,
"network.proxy.no_proxies_on": "localhost, 127.0.0.1",
}
profile.set_preferences(proxy)
device.push(profile.profile, device_profile)
device.chmod(device_storage, recursive=True)
app_args = [
"-profile",
device_profile,
"--marionette",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
]
# start app
app_name = "org.mozilla.geckoview_example"
activity_name = "GeckoViewActivity"
device.stop_application(app_name)
device.launch_activity(
app_name,
activity_name,
extra_args=app_args,
url=url,
e10s=True,
fail_if_running=False,
)
# wait for mitmdump to finish
mitmdump_process.wait()
finally:
if mitmdump_process is None:
mitmdump_process.terminate()
exit(mitmdump_process.returncode)
def set_profile(self):
self.profile = create_profile("firefox")
print("Created profile: {}".format(self.profile.profile))
def __init__(self,
app,
binary,
run_local=False,
obj_path=None,
gecko_profile=False,
gecko_profile_interval=None,
gecko_profile_entries=None,
symbols_path=None,
host=None,
is_release_build=False,
debug_mode=False):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.config['gecko_profile'] = gecko_profile
self.config['gecko_profile_interval'] = gecko_profile_interval
self.config['gecko_profile_entries'] = gecko_profile_entries
self.config['symbols_path'] = symbols_path
self.config['host'] = host
self.config['is_release_build'] = is_release_build
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.gecko_profiler = None
self.post_startup_delay = 30000
# debug mode is currently only supported when running locally
self.debug_mode = debug_mode if self.config['run_local'] else False
# if running debug-mode reduce the pause after browser startup
if self.debug_mode:
self.post_startup_delay = 3000
self.log.info(
"debug-mode enabled, reducing post-browser startup pause to %d ms"
% self.post_startup_delay)
# Create the profile; for geckoview we want a firefox profile type
if self.config['app'] == 'geckoview':
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Clear any existing mozilla.cfg file to prevent earlier
# runs using mitmproxy from interfering with settings.
remove_autoconfig(binary)
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] == "geckoview":
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBAndroid(verbose=True)
self.device.clear_logcat()
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
self.log.info("raptor config: %s" % str(self.config))
def __init__(self,
app,
binary,
run_local=False,
obj_path=None,
gecko_profile=False,
gecko_profile_interval=None,
gecko_profile_entries=None,
symbols_path=None,
host=None,
power_test=False,
is_release_build=False,
debug_mode=False):
# Override the magic --host HOST_IP with the value of the environment variable.
if host == 'HOST_IP':
host = os.environ['HOST_IP']
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.config['gecko_profile'] = gecko_profile
self.config['gecko_profile_interval'] = gecko_profile_interval
self.config['gecko_profile_entries'] = gecko_profile_entries
self.config['symbols_path'] = symbols_path
self.config['host'] = host
self.config['power_test'] = power_test
self.config['is_release_build'] = is_release_build
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.gecko_profiler = None
self.post_startup_delay = 30000
self.device = None
# debug mode is currently only supported when running locally
self.debug_mode = debug_mode if self.config['run_local'] else False
# if running debug-mode reduce the pause after browser startup
if self.debug_mode:
self.post_startup_delay = 3000
self.log.info(
"debug-mode enabled, reducing post-browser startup pause to %d ms"
% self.post_startup_delay)
# Create the profile; for geckoview/fennec we want a firefox profile type
if self.config['app'] in ["geckoview", "fennec"]:
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# add profile dir to our config
self.config['local_profile_dir'] = self.profile.profile
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] in ["geckoview", "fennec"]:
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBDevice(verbose=True)
self.device.clear_logcat()
if self.config['power_test']:
init_geckoview_power_test(self)
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args,
symbols_path=self.config['symbols_path'])
self.log.info("raptor config: %s" % str(self.config))
def start(self, url="about:blank"):
# create profile
profile = create_profile("firefox")
print("Created profile: {}".format(profile.profile))
# create certificate database
certdb = "sql:{}/".format(profile.profile)
print("Creating certificate database")
command = [self.certutil, "-N", "-v", "-d", certdb, "--empty-password"]
subprocess.call(command)
# install mitmproxy certificate
command = [
self.certutil,
"-A",
"-d",
certdb,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
self.proxy.cert,
]
print("Installing {} into certificate database".format(
self.proxy.cert))
subprocess.call(command)
# verify certificate is installed
command = [self.certutil, "-d", certdb, "-L"]
assert "mitmproxy-cert" in subprocess.check_output(command)
# setup device
device = ADBAndroid()
device.shell("pm clear {}".format(self.APP_NAME))
device.create_socket_connection("reverse", "tcp:8080", "tcp:8080")
device_storage = "/sdcard/raptor"
device_profile = os.path.join(device_storage, "profile")
if device.is_dir(device_storage):
device.rm(device_storage, recursive=True)
device.mkdir(device_storage)
device.mkdir(device_profile)
userjs = os.path.join(profile.profile, "user.js")
with open(userjs) as f:
prefs = f.readlines()
prefs = [p for p in prefs if "network.proxy" not in p]
with open(userjs, "w") as f:
f.writelines(prefs)
profile.set_preferences({
"network.proxy.type":
1,
"network.proxy.http":
"127.0.0.1",
"network.proxy.http_port":
8080,
"network.proxy.ssl":
"127.0.0.1",
"network.proxy.ssl_port":
8080,
"network.proxy.no_proxies_on":
"localhost, 127.0.0.1",
})
device.push(profile.profile, device_profile)
device.chmod(device_storage, recursive=True)
app_args = [
"-profile",
device_profile,
"--marionette",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
]
# start app
device.stop_application(self.APP_NAME)
device.launch_activity(
self.APP_NAME,
self.ACTIVITY_NAME,
extra_args=app_args,
url=url,
e10s=True,
fail_if_running=False,
)