def fixture_httpd_with_docroot(num_requests):
"""Yields a started MozHttpd server with docroot set."""
@mozhttpd.handlers.json_response
def get_handler(request, objid):
"""Handler for HTTP GET requests."""
num_requests["get_handler"] += 1
return (
200,
{
"called": num_requests["get_handler"],
"id": objid,
"query": request.query,
},
)
httpd = mozhttpd.MozHttpd(
port=0,
docroot=os.path.dirname(os.path.abspath(__file__)),
urlhandlers=[{
"method": "GET",
"path": "/api/resource/([^/]+)/?",
"function": get_handler,
}],
)
httpd.start(block=False)
yield httpd
httpd.stop()
def setup_webserver(webserver):
"""use mozhttpd to setup a webserver"""
scheme = "http://"
if (webserver.startswith('http://') or webserver.startswith('chrome://')
or webserver.startswith('file:///')):
scheme = ""
elif '://' in webserver:
print "Unable to parse user defined webserver: '%s'" % (webserver)
sys.exit(2)
url = urlparse.urlparse('%s%s' % (scheme, webserver))
port = url.port
if port:
import mozhttpd
return mozhttpd.MozHttpd(host=url.hostname,
port=int(port),
docroot=here,
urlhandlers=[{
'method': 'GET',
'path': '/results',
'function': collectResults
}])
else:
print "WARNING: unable to start web server without custom port configured"
return None
def test_symbols_path_url(check_for_crashes, minidump_files):
"""Test that passing a URL as symbols_path correctly fetches the URL."""
data = {"retrieved": False}
def make_zipfile():
data = BytesIO()
z = zipfile.ZipFile(data, 'w')
z.writestr("symbols.txt", "abc/xyz")
z.close()
return data.getvalue()
def get_symbols(req):
data["retrieved"] = True
headers = {}
return (200, headers, make_zipfile())
httpd = mozhttpd.MozHttpd(port=0,
urlhandlers=[{
'method': 'GET',
'path': '/symbols',
'function': get_symbols
}])
httpd.start()
symbol_url = urlunsplit(
('http', '%s:%d' % httpd.httpd.server_address, '/symbols', '', ''))
assert 1 == check_for_crashes(symbols_path=symbol_url)
assert data["retrieved"]
def test_basic(self):
""" Test mozhttpd can serve files """
tempdir = tempfile.mkdtemp()
# sizes is a dict of the form: name -> [size, binary_string, filepath]
sizes = {'small': [128], 'large': [16384]}
for k in sizes.keys():
# Generate random binary string
sizes[k].append(os.urandom(sizes[k][0]))
# Add path of file with binary string to list
fpath = os.path.join(tempdir, k)
sizes[k].append(fpath)
# Write binary string to file
with open(fpath, 'wb') as f:
f.write(sizes[k][1])
server = mozhttpd.MozHttpd(docroot=tempdir)
server.start()
server_url = server.get_url()
# Retrieve file and check contents matchup
for k in sizes.keys():
retrieved_content = mozfile.load(server_url + k).read()
self.assertEqual(retrieved_content, sizes[k][1])
# Cleanup tempdir and related files
mozfile.rmtree(tempdir)
def start(self):
self.stop()
self.servers = []
port = self.port
num_errors = 0
while len(self.servers) < self.count:
self.servers.append(
mozhttpd.MozHttpd(host=self.host,
port=port,
docroot=self.docroot))
try:
self.servers[-1].start()
except socket.error as error:
if isinstance(error, socket.error):
if error.errno == 98:
print("port {} is in use.".format(port))
else:
print("port {} error {}".format(port, error))
elif isinstance(error, str):
print("port {} error {}".format(port, error))
self.servers.pop()
num_errors += 1
except Exception as error:
print("port {} error {}".format(port, error))
self.servers.pop()
num_errors += 1
if num_errors > 15:
raise Exception("Too many errors in webservers.py")
port += 1
def start(self):
self.stop()
self.servers = []
port = self.port
while len(self.servers) < self.count:
self.servers.append(
mozhttpd.MozHttpd(host=self.host,
port=port,
docroot=self.docroot))
try:
self.servers[-1].start()
except socket.error as error:
if isinstance(error, socket.error):
if error.errno == 98:
print("port {} is in use.").format(port)
else:
print("port {} error {}").format(port, error)
elif isinstance(error, str):
print("port {} error {}").format(port, error)
self.servers.pop()
except Exception as error:
print("port {} error {}").format(port, error)
self.servers.pop()
port += 1
def test_symbol_path_url(self):
"""
Test that passing a URL as symbols_path correctly fetches the URL.
"""
open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo")
self.stdouts.append(["this is some output"])
def make_zipfile():
data = StringIO.StringIO()
z = zipfile.ZipFile(data, 'w')
z.writestr("symbols.txt", "abc/xyz")
z.close()
return data.getvalue()
def get_symbols(req):
headers = {}
return (200, headers, make_zipfile())
httpd = mozhttpd.MozHttpd(port=0,
urlhandlers=[{
'method': 'GET',
'path': '/symbols',
'function': get_symbols
}])
httpd.start()
symbol_url = urlparse.urlunsplit(
('http', '%s:%d' % httpd.httpd.server_address, '/symbols', '', ''))
self.assert_(
mozcrash.check_for_crashes(self.tempdir,
symbol_url,
stackwalk_binary=self.stackwalk,
quiet=True))
def test_no_docroot(self):
"""Test that path mappings with no docroot work."""
with TemporaryDirectory() as d1:
httpd = mozhttpd.MozHttpd(port=0, path_mappings={'/foo': d1})
httpd.start(block=False)
self.try_get_expect_404(httpd.get_url())
httpd.stop()
def setup_webserver(webserver):
"""use mozhttpd to setup a webserver"""
scheme = "http://"
if (webserver.startswith('http://') or webserver.startswith('chrome://')
or webserver.startswith('file:///')): # noqa
scheme = ""
elif '://' in webserver:
print "Unable to parse user defined webserver: '%s'" % (webserver)
sys.exit(2)
url = urlparse.urlparse('%s%s' % (scheme, webserver))
port = url.port
if port:
import mozhttpd
return mozhttpd.MozHttpd(host=url.hostname,
port=int(port),
docroot=here)
else:
print(
"WARNING: unable to start web server without custom port"
" configured")
return None
def build_docs(self, what=None, format=None, outdir=None, auto_open=True, http=None):
self._activate_virtualenv()
self.virtualenv_manager.install_pip_package('sphinx_rtd_theme==0.1.6')
import sphinx
import webbrowser
if not outdir:
outdir = os.path.join(self.topobjdir, 'docs')
if not what:
what = [os.path.join(self.topsrcdir, 'tools')]
outdir = os.path.join(outdir, format)
generated = []
failed = []
for path in what:
path = os.path.normpath(os.path.abspath(path))
docdir = self._find_doc_dir(path)
if not docdir:
failed.append((path, 'could not find docs at this location'))
continue
# find project name to use as a namespace within `outdir`
project = self._find_project_name(docdir)
savedir = os.path.join(outdir, project)
args = [
'sphinx',
'-b', format,
docdir,
savedir,
]
result = sphinx.build_main(args)
if result != 0:
failed.append((path, 'sphinx return code %d' % result))
else:
generated.append(savedir)
index_path = os.path.join(savedir, 'index.html')
if not http and auto_open and os.path.isfile(index_path):
webbrowser.open(index_path)
if generated:
print('\nGenerated documentation:\n%s\n' % '\n'.join(generated))
if failed:
failed = ['%s: %s' % (f[0], f[1]) for f in failed]
return die('failed to generate documentation:\n%s' % '\n'.join(failed))
if http is not None:
host, port = http.split(':', 1)
addr = (host, int(port))
if len(addr) != 2:
return die('invalid address: %s' % http)
httpd = mozhttpd.MozHttpd(host=addr[0], port=addr[1], docroot=outdir)
print('listening on %s:%d' % addr)
httpd.start(block=True)
def check_logging(self, log_requests=False):
httpd = mozhttpd.MozHttpd(port=0, docroot=here, log_requests=log_requests)
httpd.start(block=False)
url = "http://%s:%s/" % ('127.0.0.1', httpd.httpd.server_port)
f = urllib2.urlopen(url)
f.read()
return httpd.request_log
def run_capture(options, capture_file):
device_prefs = eideticker.getDevicePrefs(options)
capture_server = CaptureServer(capture_file,
options.capture_device,
options.mode,
capture=options.capture)
host = moznetwork.get_ip()
docroot = eideticker.runtest.TEST_DIR
httpd = mozhttpd.MozHttpd(port=0,
host=host,
docroot=docroot,
urlhandlers=[{
'method':
'GET',
'path':
'/api/captures/start/?',
'function':
capture_server.start_capture
}, {
'method':
'GET',
'path':
'/api/captures/end/?',
'function':
capture_server.end_capture
}])
httpd.start(block=False)
print "Serving '%s' at %s:%s" % (httpd.docroot, httpd.host,
httpd.httpd.server_port)
device = eideticker.getDevice(**device_prefs)
mode = options.mode
if not mode:
mode = device.hdmiResolution
url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port)
device.executeCommand("tap", [100, 100])
if device_prefs['devicetype'] == 'android':
device.launchFennec(options.appname, url=url)
else:
if not options.wifi_settings_file:
print "WIFI settings file (see --help) required for B2G!"
sys.exit(1)
device.restartB2G()
device.connectWIFI(json.loads(open(options.wifi_settings_file).read()))
device.marionette.execute_script("window.location.href='%s';" % url)
while not capture_server.finished:
time.sleep(0.25)
capture_server.convert_capture()
httpd.stop()
def fixture_httpd_no_docroot(tmpdir):
d1 = tmpdir.mkdir("d1")
httpd = mozhttpd.MozHttpd(
port=0,
path_mappings={"/foo": str(d1)},
)
httpd.start(block=False)
yield httpd
httpd.stop()
d1.remove()
def test_base_url(self):
httpd = mozhttpd.MozHttpd(port=0)
self.assertEqual(httpd.get_url(), None)
httpd.start(block=False)
self.assertEqual("http://127.0.0.1:%s/" % httpd.httpd.server_port,
httpd.get_url())
self.assertEqual(
"http://127.0.0.1:%s/cheezburgers.html" % httpd.httpd.server_port,
httpd.get_url(path="/cheezburgers.html"))
httpd.stop()
def __init__(self, testinfo, actions={}, docroot=None, **kwargs):
super(WebTest, self).__init__(testinfo,
track_start_frame=True,
track_end_frame=True,
**kwargs)
self.actions = actions
self.capture_server = CaptureServer(self)
self.host = moznetwork.get_ip()
self.http = mozhttpd.MozHttpd(docroot=docroot,
host=self.host,
port=0,
log_requests=True,
urlhandlers=[{
'method':
'GET',
'path':
'/api/captures/start/?',
'function':
self.capture_server.start_capture
}, {
'method':
'GET',
'path':
'/api/captures/end/?',
'function':
self.capture_server.end_capture
}, {
'method':
'POST',
'path':
'/api/captures/input/?',
'function':
self.capture_server.input
}])
self.http.start(block=False)
connected = False
tries = 0
while not connected and tries < 20:
tries += 1
import socket
s = socket.socket()
try:
s.connect((self.host, self.http.httpd.server_port))
connected = True
except Exception:
self.log("Can't connect to %s:%s, retrying..." %
(self.host, self.http.httpd.server_port))
self.log("Test URL is: %s" % self.url)
if not connected:
raise "Could not open webserver. Error!"
def test_multipart_path_mapping(self):
"""Test that a path mapping with multiple directories works."""
with TemporaryDirectory() as d1:
open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents")
httpd = mozhttpd.MozHttpd(port=0,
path_mappings={'/abc/def/ghi': d1})
httpd.start(block=False)
self.try_get(httpd.get_url("/abc/def/ghi/test1.txt"),
"test 1 contents")
self.try_get_expect_404(httpd.get_url("/abc/test1.txt"))
self.try_get_expect_404(httpd.get_url("/abc/def/test1.txt"))
httpd.stop()
def fixture_httpd_multipart_path_mapping(tmpdir):
d1 = tmpdir.mkdir("d1")
d1.join("test1.txt").write("test 1 contents")
httpd = mozhttpd.MozHttpd(
port=0,
path_mappings={"/abc/def/ghi": str(d1)},
)
httpd.start(block=False)
yield httpd
httpd.stop()
d1.remove()
def test_basic(self):
"""Test that requests to docroot and a path mapping work as expected."""
with TemporaryDirectory() as d1, TemporaryDirectory() as d2:
open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents")
open(os.path.join(d2, "test2.txt"), "w").write("test 2 contents")
httpd = mozhttpd.MozHttpd(port=0,
docroot=d1,
path_mappings={'/files': d2})
httpd.start(block=False)
self.try_get(httpd.get_url("/test1.txt"), "test 1 contents")
self.try_get(httpd.get_url("/files/test2.txt"), "test 2 contents")
self.try_get_expect_404(httpd.get_url("/files/test2_nope.txt"))
httpd.stop()
def test_install_webextension_sans_id(self):
server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
server.start()
try:
addon = server.get_url() + 'apply-css-sans-id.xpi'
self.am.install_from_path(addon)
finally:
server.stop()
self.assertEqual(len(self.am.downloaded_addons), 1)
self.assertTrue(os.path.isfile(self.am.downloaded_addons[0]))
self.assertIn('temporary-addon.xpi',
os.path.basename(self.am.downloaded_addons[0]))
def test_install_from_path_url(self):
server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
server.start()
addon = server.get_url() + 'empty.xpi'
self.am.install_from_path(addon)
server.stop()
self.assertEqual(len(self.am.downloaded_addons), 1)
self.assertTrue(os.path.isfile(self.am.downloaded_addons[0]))
self.assertIn('*****@*****.**',
os.path.basename(self.am.downloaded_addons[0]))
def test_substring_mappings(self):
"""Test that a path mapping that's a substring of another works."""
with TemporaryDirectory() as d1, TemporaryDirectory() as d2:
open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents")
open(os.path.join(d2, "test2.txt"), "w").write("test 2 contents")
httpd = mozhttpd.MozHttpd(port=0,
path_mappings={'/abcxyz': d1,
'/abc': d2, }
)
httpd.start(block=False)
self.try_get(httpd.get_url("/abcxyz/test1.txt"), "test 1 contents")
self.try_get(httpd.get_url("/abc/test2.txt"), "test 2 contents")
httpd.stop()
def test_nonexistent_resources(self):
# Create a server with a placeholder handler so we don't fall back
# to serving local files
httpd = mozhttpd.MozHttpd(port=0)
httpd.start(block=False)
server_port = httpd.httpd.server_port
# GET: Return 404 for non-existent endpoint
exception_thrown = False
try:
urllib2.urlopen(self.get_url('/api/resource/', server_port, None))
except urllib2.HTTPError, e:
self.assertEqual(e.code, 404)
exception_thrown = True
def test_install_from_path_url(self):
server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
server.start()
addon = server.get_url() + 'empty.xpi'
self.am.install_from_path(addon)
# bug 932337
# We currently store downloaded add-ons with a tmp filename.
# So we cannot successfully do real comparisons
self.assertEqual(self.am.installed_addons, self.am.downloaded_addons)
for addon in self.am.downloaded_addons:
self.assertTrue(os.path.isfile(addon))
def check_filelisting(self, path=''):
filelist = os.listdir(here)
httpd = mozhttpd.MozHttpd(port=0, docroot=here)
httpd.start(block=False)
f = urllib2.urlopen("http://%s:%s/%s" % ('127.0.0.1', httpd.httpd.server_port, path))
for line in f.readlines():
webline = re.sub('\<[a-zA-Z0-9\-\_\.\=\"\'\/\\\%\!\@\#\$\^\&\*\(\) ]*\>', '', line.strip('\n')).strip('/').strip().strip('@')
if webline and not webline.startswith("Directory listing for"):
self.assertTrue(webline in filelist,
"File %s in dir listing corresponds to a file" % webline)
filelist.remove(webline)
self.assertFalse(filelist, "Should have no items in filelist (%s) unaccounted for" % filelist)
def test_download(self):
server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
server.start()
# Download a valid add-on without a class instance to the general
# tmp folder and clean-up
try:
addon = server.get_url() + 'empty.xpi'
xpi_file = mozprofile.addons.AddonManager.download(addon)
self.assertTrue(os.path.isfile(xpi_file))
self.assertIn('*****@*****.**',
os.path.basename(xpi_file))
self.assertNotIn(self.tmpdir, os.path.dirname(xpi_file))
finally:
# Given that the file is stored outside of the created tmp dir
# we have to ensure to explicitely remove it
if os.path.isfile(xpi_file):
os.remove(xpi_file)
# Download an valid add-on to a special folder
addon = server.get_url() + 'empty.xpi'
xpi_file = self.am.download(addon, self.tmpdir)
self.assertTrue(os.path.isfile(xpi_file))
self.assertIn('*****@*****.**',
os.path.basename(xpi_file))
self.assertIn(self.tmpdir, os.path.dirname(xpi_file))
self.assertEqual(self.am.downloaded_addons, [])
os.remove(xpi_file)
# Download an invalid add-on to a special folder
addon = server.get_url() + 'invalid.xpi'
self.assertRaises(mozprofile.addons.AddonFormatError,
self.am.download, addon, self.tmpdir)
self.assertEqual(os.listdir(self.tmpdir), [])
# Download from an invalid URL
addon = server.get_url() + 'not_existent.xpi'
self.assertRaises(error.HTTPError,
self.am.download, addon, self.tmpdir)
self.assertEqual(os.listdir(self.tmpdir), [])
# Download from an invalid URL
addon = 'not_existent.xpi'
self.assertRaises(ValueError,
self.am.download, addon, self.tmpdir)
self.assertEqual(os.listdir(self.tmpdir), [])
server.stop()
def run_server(doc_root):
httpd_server = mozhttpd.MozHttpd(port=16932,
docroot=doc_root,
urlhandlers=[{
'method': 'GET',
'path': '/audio/',
'function': parseGETUrl
}, {
'method': 'GET',
'path': '/server/?',
'function': parseGETUrl
}])
talos.utils.info("Server %s at %s:%s", httpd_server.docroot,
httpd_server.host, httpd_server.port)
ObjectDb.httpd_server = httpd_server
httpd_server.start(block=True)
def test_noclean(self):
"""test `restore=True/False` functionality"""
server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
server.start()
profile = tempfile.mkdtemp()
tmpdir = tempfile.mkdtemp()
try:
# empty initially
self.assertFalse(bool(os.listdir(profile)))
# make an addon
addons = []
addons.append(
generate_addon('*****@*****.**', path=tmpdir))
addons.append(server.get_url() + 'empty.xpi')
# install it with a restore=True AddonManager
am = mozprofile.addons.AddonManager(profile, restore=True)
for addon in addons:
am.install_from_path(addon)
# now its there
self.assertEqual(os.listdir(profile), ['extensions'])
staging_folder = os.path.join(profile, 'extensions', 'staged')
self.assertTrue(os.path.exists(staging_folder))
self.assertEqual(len(os.listdir(staging_folder)), 2)
# del addons; now its gone though the directory tree exists
downloaded_addons = am.downloaded_addons
del am
self.assertEqual(os.listdir(profile), ['extensions'])
self.assertTrue(os.path.exists(staging_folder))
self.assertEqual(os.listdir(staging_folder), [])
for addon in downloaded_addons:
self.assertFalse(os.path.isfile(addon))
finally:
mozfile.rmtree(tmpdir)
mozfile.rmtree(profile)
def run_server(doc_root):
ObjectDb.audio_utils = media_utils.AudioUtils()
httpd_server = mozhttpd.MozHttpd(
port=16932,
docroot=doc_root,
urlhandlers=[
{'method': 'GET', 'path': '/audio/', 'function': parseGETUrl},
{'method': 'GET', 'path': '/server/?', 'function': parseGETUrl}
]
)
logging.info("Server %s at %s:%s",
httpd_server.docroot, httpd_server.host,
httpd_server.port)
ObjectDb.httpd_server = httpd_server
httpd_server.start()
return httpd_server
def fixture_httpd_basic(tmpdir):
d1 = tmpdir.mkdir("d1")
d1.join("test1.txt").write("test 1 contents")
d2 = tmpdir.mkdir("d2")
d2.join("test2.txt").write("test 2 contents")
httpd = mozhttpd.MozHttpd(
port=0,
docroot=str(d1),
path_mappings={"/files": str(d2)},
)
httpd.start(block=False)
yield httpd
httpd.stop()
d1.remove()
d2.remove()
def fixture_httpd_substring_mappings(tmpdir):
d1 = tmpdir.mkdir("d1")
d1.join("test1.txt").write("test 1 contents")
d2 = tmpdir.mkdir("d2")
d2.join("test2.txt").write("test 2 contents")
httpd = mozhttpd.MozHttpd(
port=0,
path_mappings={
"/abcxyz": str(d1),
"/abc": str(d2)
},
)
httpd.start(block=False)
yield httpd
httpd.stop()
d1.remove()
d2.remove()