def test_get_playback(get_binary):
config["platform"] = mozinfo.os
if "win" in config["platform"]:
# this test is not yet supported on windows
assert True
return
config["obj_path"] = os.path.dirname(get_binary("firefox"))
config["playback_tool"] = "mitmproxy"
config["playback_version"] = "5.1.1"
config["playback_files"] = [
os.path.join(
os.path.dirname(os.path.abspath(os.path.dirname(__file__))),
"raptor",
"tooltool-manifests",
"playback",
"mitm5-linux-firefox-amazon.manifest",
)
]
config["binary"] = get_binary("firefox")
config["run_local"] = run_local
config["app"] = "firefox"
config["host"] = "127.0.0.1"
playback = get_playback(config)
assert isinstance(playback, MitmproxyDesktop)
playback.start()
time.sleep(1)
playback.stop()
def test_mitm_check_proxy(*args):
# test setup
pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest")
config = {
"playback_tool": "mitmproxy",
"playback_files": [os.path.join(here, "files", pageset_name)],
"playback_version": "5.1.1",
"platform": mozinfo.os,
"run_local": "MOZ_AUTOMATION" not in os.environ,
"binary": "firefox",
"app": "firefox",
"host": "127.0.0.1",
}
with tempdir() as obj_path:
config["obj_path"] = obj_path
playback = get_playback(config)
assert playback is not None
try:
playback.start()
url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js"
assert get_status_code(url, playback) == 200
url = "http://mozproxy/checkProxy"
assert get_status_code(url, playback) == 404
finally:
playback.stop()
def test_mitm_check_proxy(*args):
# test setup
bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
pageset_name = "mitm4-linux-firefox-amazon.manifest"
playback_recordings = "amazon.mp"
config = {
"playback_tool": "mitmproxy",
"playback_binary_manifest": bin_name,
"playback_pageset_manifest": os.path.join(here, "files", pageset_name),
"playback_version": '4.0.4',
"platform": mozinfo.os,
"run_local": "MOZ_AUTOMATION" not in os.environ,
"binary": "firefox",
"app": "firefox",
"host": "127.0.0.1",
}
with tempdir() as obj_path:
config["obj_path"] = obj_path
playback = get_playback(config)
playback.config['playback_files'] = [
os.path.join(obj_path, "testing", "mozproxy", playback_recordings)]
assert playback is not None
try:
playback.start()
url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js"
assert get_status_code(url, playback) == 200
url = "http://mozproxy/checkProxy"
assert get_status_code(url, playback) == 404
finally:
playback.stop()
def test_get_playback(get_binary):
config['platform'] = mozinfo.os
if 'win' in config['platform']:
# this test is not yet supported on windows
assert True
return
config['obj_path'] = os.path.dirname(get_binary('firefox'))
config['playback_tool'] = 'mitmproxy'
config['playback_version'] = '4.0.4'
config[
'playback_binary_manifest'] = 'mitmproxy-rel-bin-4.0.4-{platform}.manifest'
config['playback_pageset_manifest'] = os.path.join(
os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "raptor",
"playback", 'mitm4-linux-firefox-amazon.manifest')
config['playback_recordings'] = 'amazon.mp'
config['binary'] = get_binary('firefox')
config['run_local'] = run_local
config['app'] = 'firefox'
config['host'] = 'https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1'
playback = get_playback(config)
playback.config['playback_files'] = [
os.path.join(playback.mozproxy_dir, config['playback_recordings'])
]
assert isinstance(playback, MitmproxyDesktop)
playback.start()
time.sleep(1)
playback.stop()
def __call__(self, metadata):
self.metadata = metadata
# replace with artifacts
config = {
"run_local": True,
"playback_tool": "mitmproxy",
"host": "localhost",
"binary": self.mach_cmd.get_binary_path(),
"obj_path": self.mach_cmd.topobjdir,
"platform": mozinfo.os,
"playback_files": [os.path.join(HERE, "example.dump")],
"app": "firefox",
}
self.info("setting up the proxy")
self.proxy = get_playback(config)
if self.proxy is not None:
self.proxy.start()
port = str(self.proxy.port)
prefs = {}
prefs["network.proxy.type"] = 1
prefs["network.proxy.http"] = "localhost"
prefs["network.proxy.http_port"] = port
prefs["network.proxy.ssl"] = "localhost"
prefs["network.proxy.ssl_port"] = port
prefs["network.proxy.no_proxies_on"] = "localhost"
metadata["browser"]["prefs"].update(prefs)
return metadata
def start_playback(self, test):
# creating the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config, self.device)
# finish its configuration
script = os.path.join(here, "playback", "alternate-server-replay.py")
recordings = test.get("playback_recordings")
if recordings:
script_args = []
proxy_dir = self.playback.mozproxy_dir
for recording in recordings.split():
if not recording:
continue
script_args.append(os.path.join(proxy_dir, recording))
script = '""%s %s""' % (script, " ".join(script_args))
# this part is platform-specific
if mozinfo.os == "win":
script = script.replace("\\", "\\\\\\")
self.playback.config['playback_tool_args'] = ["-s", script]
# let's start it!
self.playback.start()
# for android we must make the playback server available to the device
if self.config['app'] in self.firefox_android_apps and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor playback server port available to device")
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
def test_mitm(*args):
bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
pageset_name = "mitm4-linux-firefox-amazon.manifest"
config = {
"playback_tool": "mitmproxy",
"playback_binary_manifest": bin_name,
"playback_pageset_manifest": pageset_name,
"playback_version": '4.0.4',
"platform": mozinfo.os,
"playback_recordings": os.path.join(here, "paypal.mp"),
"run_local": True,
"binary": "firefox",
"app": "firefox",
"host": "example.com",
}
with tempdir() as obj_path:
config["obj_path"] = obj_path
playback = get_playback(config)
playback.config['playback_files'] = config['playback_recordings']
assert playback is not None
try:
playback.start()
finally:
playback.stop()
def start_playback(self, test):
# creating the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config, self.device)
self.get_proxy_command_for_mitm(test, self.config['playback_version'])
# let's start it!
self.playback.start()
def setupTp6(self):
# tp5n stores its manifest in the zip file that gets extracted, tp6
# doesn't so we just keep one in our project dir for now.
default_tp6_pages_manifest = os.path.join(AWSY_PATH, 'conf',
'tp6-pages.yml')
tp6_pages_manifest = self.testvars.get("pageManifest",
default_tp6_pages_manifest)
urls = []
recordings = set()
with open(tp6_pages_manifest) as f:
d = yaml.safe_load(f)
for r in d:
recordings.add(r['rec'])
url = r['url']
if isinstance(url, list):
urls.extend(url)
else:
urls.append(url)
self._urls = urls
# Indicate that we're using tp6 in the perf data.
self._extra_opts = ["tp6"]
if self.marionette.get_pref('fission.autostart'):
self._extra_opts.append("fission")
# Now we setup the mitm proxy with our tp6 pageset.
tp6_pageset_manifest = os.path.join(AWSY_PATH, 'tp6-pageset.manifest')
config = {
'playback_tool': 'mitmproxy',
'playback_version': '4.0.4',
'playback_pageset_manifest': tp6_pageset_manifest,
'platform': mozinfo.os,
'obj_path': self._webroot_dir,
'binary': self._binary,
'run_local': self._run_local,
'app': 'firefox',
'host': '127.0.0.1',
'ignore_mitmdump_exit_failure': True,
}
self._playback = get_playback(config)
self._playback.start()
# We need to reload after the mitmproxy cert is installed
self.marionette.restart(clean=False)
# Setup WebDriver capabilities that we need
self.marionette.delete_session()
caps = {
"unhandledPromptBehavior":
"dismiss", # Ignore page navigation warnings
}
self.marionette.start_session(caps)
self.marionette.set_context('chrome')
def setupTp6(self):
# tp5n stores its manifest in the zip file that gets extracted, tp6
# doesn't so we just keep one in our project dir for now.
default_tp6_pages_manifest = os.path.join(AWSY_PATH, "conf",
"tp6-pages.yml")
tp6_pages_manifest = self.testvars.get("pageManifest",
default_tp6_pages_manifest)
urls = []
recordings = set()
with open(tp6_pages_manifest) as f:
d = yaml.safe_load(f)
for r in d:
recordings.add(r["rec"])
url = r["url"]
if isinstance(url, list):
urls.extend(url)
else:
urls.append(url)
self._urls = urls
# Indicate that we're using tp6 in the perf data.
self._extra_opts = ["tp6"]
if self.marionette.get_pref("fission.autostart"):
self._extra_opts.append("fission")
# Now we setup the mitm proxy with our tp6 pageset.
tp6_pageset_manifest = os.path.join(AWSY_PATH, "tp6-pageset.manifest")
config = {
"playback_tool": "mitmproxy",
"playback_version": "4.0.4",
"playback_files": [tp6_pageset_manifest],
"platform": mozinfo.os,
"obj_path": self._webroot_dir,
"binary": self._binary,
"run_local": self._run_local,
"app": "firefox",
"host": "127.0.0.1",
"ignore_mitmdump_exit_failure": True,
}
self._playback = get_playback(config)
self._playback.start()
# We need to reload after the mitmproxy cert is installed
self.marionette.restart(clean=False)
# Setup WebDriver capabilities that we need
self.marionette.delete_session()
caps = {
"unhandledPromptBehavior":
"dismiss", # Ignore page navigation warnings
}
self.marionette.start_session(caps)
self.marionette.set_context("chrome")
def test_record_and_replay(*args):
# test setup
recording_file = os.path.join(here, "files", "new_recoding.zip")
# Record part
config = {
"playback_tool": "mitmproxy",
"recording_file": recording_file,
"playback_version": "5.1.1",
"platform": mozinfo.os,
"run_local": "MOZ_AUTOMATION" not in os.environ,
"binary": "firefox",
"app": "firefox",
"host": "127.0.0.1",
"record": True,
}
with tempdir() as obj_path:
config["obj_path"] = obj_path
record = get_playback(config)
assert record is not None
try:
record.start()
url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js"
assert get_status_code(url, record) == 200
finally:
record.stop()
# playback part
config["record"] = False
config["recording_file"] = None
config["playback_files"] = [recording_file]
playback = get_playback(config)
assert playback is not None
try:
playback.start()
url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js"
assert get_status_code(url, playback) == 200
finally:
playback.stop()
def start_playback(self, test):
# creating the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config)
self.playback.config["playback_files"] = self.get_recording_paths(test)
# let's start it!
self.playback.start()
self.log_recording_dates(test)
def start_playback(self, test):
# startup the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config, self.device)
# for android we must make the playback server available to the device
if self.config['app'] in self.firefox_android_apps and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info("making the raptor playback server port available to device")
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
def using_proxy(self, video_id):
config = {}
config["binary"] = self.marionette.bin
config["app"] = "firefox"
config["platform"] = mozinfo.os
config["processor"] = mozinfo.processor
config["run_local"] = "MOZ_UPLOAD_DIR" not in os.environ
if "MOZ_UPLOAD_DIR" not in os.environ:
config["obj_path"] = os.environ["OBJ_PATH"]
playback_dir = os.path.join(config["obj_path"], "testing",
"mozproxy")
else:
root_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]))
playback_dir = os.path.join(root_dir, "testing", "mozproxy")
config["host"] = "localhost"
config["playback_tool"] = "mitmproxy"
config["playback_artifacts"] = os.path.join(here,
"%s.manifest" % video_id)
# XXX once Bug 1540622 lands, we can use the version here
# config["playback_version"] = "4.0.4"
# and have playback_binary_manifest default to
# mitmproxy-rel-bin-{playback_version}-{platform}.manifest
# so we don't have to ask amozproxy tool user to provide this:
config[
"playback_binary_manifest"] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
playback_file = os.path.join(playback_dir, "%s.playback" % video_id)
config["playback_tool_args"] = [
"--set",
"stream_large_bodies=30",
"--ssl-insecure",
"--server-replay-nopop",
"--set",
"upstream_cert=false",
"-S",
playback_file,
"-s",
playback_script,
video_id,
]
proxy = get_playback(config)
if proxy is None:
raise Exception("Could not start Proxy")
proxy.start()
try:
yield proxy
finally:
proxy.stop()
def test_playback_setup_failed(*args):
class SetupFailed(Exception):
pass
def setup(*args, **kw):
def _s(self):
raise SetupFailed("Failed")
return _s
mitm.MITMDUMP_SLEEP = 0.1
bin_name = "mitmproxy-rel-bin-{platform}.manifest"
pageset_name = "mitmproxy-recordings-raptor-paypal.manifest"
config = {
"playback_tool": "mitmproxy",
"playback_binary_manifest": bin_name,
"playback_pageset_manifest": pageset_name,
"platform": mozinfo.os,
"playback_recordings": os.path.join(here, "paypal.mp"),
"run_local": True,
"obj_path": here, # XXX tmp?
"binary": "firefox",
"app": "firefox",
"host": "example.com",
}
prefix = "mozproxy.backends.mitm.MitmproxyDesktop."
with mock.patch(prefix + "setup", new_callable=setup):
with mock.patch(prefix + "stop_mitmproxy_playback") as p:
try:
get_playback(config)
except SetupFailed:
assert p.call_count == 1
except Exception:
raise
def youtube_video(video_id):
config["playback_recordings"] = "%s.playback" % video_id
config["playback_tool_args"] = [
"--set", "upstream_cert=false",
"-S", "/tmp/testing/mozproxy/%s.playback" % video_id,
]
proxy = get_playback(config)
if proxy is None:
raise Exception("Could not start Proxy")
try:
prefs = {"media.autoplay.default": 0}
prefs["browser.newtabpage.activity-stream.feeds.snippets"] = 0
prefs["browser.newtabpage.activity-stream.disableSnippets"] = 1
prefs["network.proxy.type"] = 1
prefs["network.proxy.http"] = config["host"]
prefs["network.proxy.http_port"] = 8080
prefs["network.proxy.ssl"] = config["host"]
prefs["network.proxy.ssl_port"] = 8080
prefs["network.proxy.no_proxies_on"] = config["host"]
profile = FirefoxProfile(profile="/tmp/mozprof", preferences=prefs, addons=[])
browser = apps["fxdesktop"].create(
profile=profile, app="fxdesktop", bin=config["binary"],
app_args=["about:blank"]
)
browser.start()
except Exception:
proxy.stop()
raise
try:
page = YoutubePage(video_id)
except Exception:
try:
browser.close()
finally:
proxy.stop()
raise
try:
yield page
finally:
try:
page.close()
browser.close()
finally:
proxy.stop()
def test_get_playback(get_binary):
config['platform'] = mozinfo.os
if 'win' in config['platform']:
# this test is not yet supported on windows
assert True
return
config['obj_path'] = os.path.dirname(get_binary('firefox'))
config['playback_tool'] = 'mitmproxy'
config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
config[
'playback_pageset_manifest'] = 'mitmproxy-recordings-raptor-tp6-1.manifest'
config['playback_recordings'] = 'amazon.mp'
config['binary'] = get_binary('firefox')
config['run_local'] = run_local
config['app'] = 'firefox'
config['host'] = 'example.com'
playback = get_playback(config)
assert isinstance(playback, MitmproxyDesktop)
playback.stop()
def test_mitm(*args):
mitm.MITMDUMP_SLEEP = 0.1
bin_name = "mitmproxy-rel-bin-{platform}.manifest"
pageset_name = "mitmproxy-recordings-raptor-paypal.manifest"
config = {
"playback_tool": "mitmproxy",
"playback_binary_manifest": bin_name,
"playback_pageset_manifest": pageset_name,
"platform": mozinfo.os,
"playback_recordings": os.path.join(here, "paypal.mp"),
"run_local": True,
"obj_path": here, # XXX tmp?
"binary": "firefox",
"app": "firefox",
"host": "example.com",
}
playback = get_playback(config)
assert playback is not None
playback.stop()
def test_playback_setup_failed(*args):
class SetupFailed(Exception):
pass
def setup(*args, **kw):
def _s(self):
raise SetupFailed("Failed")
return _s
bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
pageset_name = "mitm4-linux-firefox-amazon.manifest"
config = {
"playback_tool": "mitmproxy",
"playback_binary_manifest": bin_name,
"playback_pageset_manifest": pageset_name,
"playback_version": '4.0.4',
"platform": mozinfo.os,
"playback_recordings": os.path.join(here, "paypal.mp"),
"run_local": True,
"binary": "firefox",
"app": "firefox",
"host": "example.com",
}
prefix = "mozproxy.backends.mitm.MitmproxyDesktop."
with tempdir() as obj_path:
config["obj_path"] = obj_path
with mock.patch(prefix + "setup", new_callable=setup):
with mock.patch(prefix + "stop_mitmproxy_playback") as p:
try:
pb = get_playback(config)
pb.config['playback_files'] = config['playback_recordings']
pb.start()
except SetupFailed:
assert p.call_count == 1
except Exception:
raise
def test_mitm_with_retry(*args):
pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest")
config = {
"playback_tool": "mitmproxy",
"playback_files": [pageset_name],
"playback_version": "5.1.1",
"platform": mozinfo.os,
"run_local": True,
"binary": "firefox",
"app": "firefox",
"host": "example.com",
}
with tempdir() as obj_path:
config["obj_path"] = obj_path
playback = get_playback(config)
assert playback is not None
try:
playback.start()
finally:
playback.stop()
def start_playback(self, test):
# creating the playback tool
playback_dir = os.path.join(here, "tooltool-manifests", "playback")
self.config.update({
"playback_tool":
test.get("playback"),
"playback_version":
test.get("playback_version", "5.1.1"),
"playback_files": [
os.path.join(playback_dir,
test.get("playback_pageset_manifest"))
],
})
LOG.info("test uses playback tool: %s " % self.config["playback_tool"])
self.playback = get_playback(self.config)
# let's start it!
self.playback.start()
def test_playback_setup_failed(*args):
class SetupFailed(Exception):
pass
def setup(*args, **kw):
def _s(self):
raise SetupFailed("Failed")
return _s
pageset_name = os.path.join(here, "files",
"mitm5-linux-firefox-amazon.manifest")
config = {
"playback_tool": "mitmproxy",
"playback_files": [pageset_name],
"playback_version": "4.0.4",
"platform": mozinfo.os,
"run_local": True,
"binary": "firefox",
"app": "firefox",
"host": "example.com",
}
prefix = "mozproxy.backends.mitm.desktop.MitmproxyDesktop."
with tempdir() as obj_path:
config["obj_path"] = obj_path
with mock.patch(prefix + "setup", new_callable=setup):
with mock.patch(prefix + "stop_mitmproxy_playback") as p:
try:
pb = get_playback(config)
pb.start()
except SetupFailed:
assert p.call_count == 1
except Exception:
raise
def setupTp6(self):
# tp5n stores its manifest in the zip file that gets extracted, tp6
# doesn't so we just keep one in our project dir for now.
default_tp6_pages_manifest = os.path.join(AWSY_PATH, 'conf',
'tp6-pages.yml')
tp6_pages_manifest = self.testvars.get("pageManifest",
default_tp6_pages_manifest)
urls = []
recordings = set()
with open(tp6_pages_manifest) as f:
d = yaml.safe_load(f)
for r in d:
recordings.add(r['rec'])
url = r['url']
if isinstance(url, list):
urls.extend(url)
else:
urls.append(url)
self._urls = urls
# Indicate that we're using tp6 in the perf data.
self._extra_opts = ["tp6"]
# Now we setup the mitm proxy with our tp6 pageset.
tp6_pageset_manifest = os.path.join(AWSY_PATH, 'tp6-pageset.manifest')
config = {
'playback_tool': 'mitmproxy',
'playback_binary_manifest':
'mitmproxy-rel-bin-{platform}.manifest',
'playback_pageset_manifest': tp6_pageset_manifest,
'platform': mozinfo.os,
'obj_path': self._webroot_dir,
'binary': self._binary,
'run_local': self._run_local,
'app': 'firefox',
'host': 'localhost',
'ignore_mitmdump_exit_failure': True,
}
self._playback = get_playback(config)
script = os.path.join(AWSY_PATH, "awsy", "alternate-server-replay.py")
recording_arg = []
for recording in recordings:
recording_arg.append(
os.path.join(self._playback.mozproxy_dir, recording))
script = '""%s %s""' % (script, " ".join(recording_arg))
if mozinfo.os == "win":
script = script.replace("\\", "\\\\\\")
# --no-upstream-cert prevents mitmproxy from needing network access to
# the upstream servers
self._playback.config['playback_tool_args'] = [
"--no-upstream-cert", "-s", script
]
self.logger.info("Using script %s" % script)
self._playback.start()
# We need to reload after the mitmproxy cert is installed
self.marionette.restart(clean=False)
# Setup WebDriver capabilities that we need
self.marionette.delete_session()
caps = {
"unhandledPromptBehavior":
"dismiss", # Ignore page navigation warnings
}
self.marionette.start_session(caps)
self.marionette.set_context('chrome')
def test_get_unsupported_playback():
config['playback_tool'] = 'unsupported'
playback = get_playback(config)
assert playback is None
def test_get_unsupported_playback():
config["playback_tool"] = "unsupported"
playback = get_playback(config)
assert playback is None
def test_get_playback_missing_tool_name():
playback = get_playback(config)
assert playback is None
