def test_local_url_detector():
"Tests that local URLs can be detected."
err = ErrorBundle()
mp = markuptester.MarkupParser(err)
tester = mp._is_url_local
assert tester("chrome://xyz/content/abc")
assert tester("chrome://whatever/")
assert tester("local.xul")
assert not tester("http://foo.bar/")
assert not tester("https://abc.def/")
assert tester(u"chrome://xyz/content/abc")
assert tester(u"chrome://whatever/")
assert tester(u"local.xul")
assert not tester(u"http://foo.bar/")
assert not tester(u"https://abc.def/")
def test_local_url_detector():
'Tests that local URLs can be detected.'
err = ErrorBundle()
mp = markuptester.MarkupParser(err)
tester = mp._is_url_local
assert tester('chrome://xyz/content/abc')
assert tester('chrome://whatever/')
assert tester('local.xul')
assert not tester('http://foo.bar/')
assert not tester('https://abc.def/')
assert tester(u'chrome://xyz/content/abc')
assert tester(u'chrome://whatever/')
assert tester(u'local.xul')
assert not tester(u'http://foo.bar/')
assert not tester(u'https://abc.def/')
def test_script_scraping():
"""Test that the scripts in a document are collected properly."""
err = ErrorBundle()
err.supported_versions = {}
parser = markuptester.MarkupParser(err, debug=True)
parser.process(
'foo.xul', """
<doc>
<!-- One to be ignored -->
<script type="text/javascript"></script>
<script src="/relative.js"></script>
<script src="chrome://namespace/absolute.js"></script>
<script src="very_relative.js"></script>
</doc>
""", 'xul')
assert parser.found_scripts == set(
['/relative.js', 'chrome://namespace/absolute.js', 'very_relative.js'])
def _test_xul_raw(data, path, should_fail=False, type_=None):
filename = path.split("/")[-1]
extension = filename.split(".")[-1]
err = ErrorBundle()
err.supported_versions = {}
if type_:
err.detected_type = type_
parser = markuptester.MarkupParser(err, debug=True)
parser.process(filename, data, extension)
print err.print_summary(verbose=True)
if should_fail:
assert err.failed()
else:
assert not err.failed(fail_on_warnings=False)
return err
def test_packed_packages(err, xpi_package=None):
'Tests XPI and JAR files for naughty content.'
processed_files = 0
pretested_files = err.get_resource('pretested_files') or []
scripts = set()
chrome = err.get_resource('chrome.manifest_nopush')
overlays = chrome.get_applicable_overlays(err) if chrome else set()
marked_scripts = err.get_resource('marked_scripts')
if not marked_scripts:
marked_scripts = set()
identified_files = err.metadata.setdefault('identified_files', {})
# Iterate each item in the package.
for name in xpi_package:
# Warn for things like __MACOSX directories and .old files.
if '__MACOSX' in name or name.split('/')[-1].startswith('.'):
err.warning(
err_id=('testcases_content', 'test_packed_packages',
'hidden_files'),
warning='Hidden files and folders flagged',
description='Hidden files and folders complicate the review '
'process and can contain sensitive information '
'about the system that generated the XPI. Please '
'modify the packaging process so that these files '
"aren't included.",
filename=name)
continue
elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS)
or name in FLAGGED_FILES):
err.warning(
err_id=('testcases_content', 'test_packaged_packages',
'flagged_files'),
warning='Flagged filename found',
description='Files were found that are either unnecessary '
'or have been included unintentionally. They '
'should be removed.',
filename=name)
continue
# Skip the file if it's in the pre-tested files resource. This skips
# things like Jetpack files.
if name in pretested_files:
continue
# Read the file from the archive if possible.
file_data = u''
try:
file_data = xpi_package.read(name)
except KeyError:
pass
except BadZipfile:
err.error(('testcases_content', 'test_packed_packages',
'jar_subpackage_corrupt'),
'Package corrupt',
'The package appears to be corrupt.',
file=xpi_package.filename)
break
if not err.for_appversions:
hash = hashlib.sha256(file_data).hexdigest()
identified = hash_library.get(hash)
if identified is not None:
identified_files[name] = {'path': identified}
err.notice(
err_id=('testcases_content', 'test_packed_packages',
'blacklisted_js_library'),
notice='JS Library Detected',
description=('JavaScript libraries are discouraged for '
'simple add-ons, but are generally '
'accepted.',
'File %r is a known JS library' % name),
filename=name)
continue
# Process the file.
name_lower = name.lower()
if name_lower.endswith(('.js', '.jsm')):
# Add the scripts to a list to be processed later.
scripts.add(name)
elif name_lower.endswith(('.xul', '.xml', '.html', '.xhtml', '.xbl')):
# Process markup files outside of _process_file so we can get
# access to information about linked scripts and such.
parser = testendpoint_markup.MarkupParser(err)
parser.process(name, file_data,
xpi_package.info(name)['extension'])
run_regex_tests(file_data, err, name)
# Make sure the name is prefixed with a forward slash.
prefixed_name = name if name.startswith('/') else '/%s' % name
# Mark scripts as pollutable if this is an overlay file and there
# are scripts to mark.
if overlays and prefixed_name in overlays and parser.found_scripts:
# Look up the chrome URL for the overlay
reversed_chrome_url = chrome.reverse_lookup(err, name)
for script in parser.found_scripts:
# Change the URL to an absolute URL.
script = _make_script_absolute(reversed_chrome_url, script)
if script:
# Mark the script as potentially pollutable.
marked_scripts.add(script)
err.save_resource('marked_scripts', marked_scripts)
else:
err.warning(err_id=('testcases_content',
'test_packed_packages',
'invalid_chrome_url'),
warning='Invalid chrome URL',
description='The referenced chrome: URL '
'could not be resolved to a '
'script file.',
filename=name)
else:
# For all other files, simply throw it at _process_file.
_process_file(err, xpi_package, name, file_data, name_lower)
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and name_lower.endswith(
('.html', '.xhtml', '.xul', '.xml', '.xbl', '.properties',
'.dtd')):
testendpoint_langpack.test_unsafe_html(err, name, file_data)
# This aids in creating unit tests.
processed_files += 1
# If there aren't any scripts in the package, just skip the next few bits.
if not scripts:
return processed_files
# Save the list of scripts, along with where to find them and the current
# validation state.
existing_scripts = err.get_resource('scripts')
if not existing_scripts:
existing_scripts = []
existing_scripts.append({
'scripts': scripts,
'package': xpi_package,
'state': err.package_stack[:]
})
err.save_resource('scripts', existing_scripts)
return processed_files
def test_packed_packages(err, xpi_package=None):
"Tests XPI and JAR files for naughty content."
processed_files = 0
pretested_files = err.get_resource("pretested_files") or []
scripts = set()
chrome = err.get_resource("chrome.manifest_nopush")
overlays = chrome.get_applicable_overlays(err) if chrome else set()
marked_scripts = err.get_resource("marked_scripts")
if not marked_scripts:
marked_scripts = set()
identified_files = err.metadata.setdefault("identified_files", {})
# Iterate each item in the package.
for name in xpi_package:
# Warn for things like __MACOSX directories and .old files.
if ("__MACOSX" in name or name.split("/")[-1].startswith(".")):
err.warning(
err_id=("testcases_content", "test_packed_packages",
"hidden_files"),
warning="Hidden files and folders flagged",
description="Hidden files and folders complicate the review "
"process and can contain sensitive information "
"about the system that generated the XPI. Please "
"modify the packaging process so that these files "
"aren't included.",
filename=name)
continue
elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS)
or name in FLAGGED_FILES):
err.warning(
err_id=("testcases_content", "test_packaged_packages",
"flagged_files"),
warning="Flagged filename found",
description="Files were found that are either unnecessary "
"or have been included unintentionally. They "
"should be removed.",
filename=name)
continue
# Skip the file if it's in the pre-tested files resource. This skips
# things like Jetpack files.
if name in pretested_files:
continue
# Read the file from the archive if possible.
file_data = u""
try:
file_data = xpi_package.read(name)
except KeyError: # pragma: no cover
pass
if not err.for_appversions:
hash = hashlib.sha256(file_data).hexdigest()
identified = hash_library.get(hash)
if identified is not None:
identified_files[name] = {"path": identified}
err.notice(err_id=("testcases_content", "test_packed_packages",
"blacklisted_js_library"),
notice="JS Library Detected",
description=[
"JavaScript libraries are discouraged for "
"simple add-ons, but are generally "
"accepted.",
"File %r is a known JS library" % name
],
filename=name)
continue
# Process the file.
processed = False
name_lower = name.lower()
if name_lower.endswith((".js", ".jsm")):
# Add the scripts to a list to be processed later.
scripts.add(name)
elif name_lower.endswith((".xul", ".xml", ".html", ".xhtml", ".xbl")):
# Process markup files outside of _process_file so we can get
# access to information about linked scripts and such.
parser = testendpoint_markup.MarkupParser(err)
parser.process(name, file_data,
xpi_package.info(name)["extension"])
run_regex_tests(file_data, err, name)
# Make sure the name is prefixed with a forward slash.
prefixed_name = name if name.startswith("/") else "/%s" % name
# Mark scripts as pollutable if this is an overlay file and there
# are scripts to mark.
if overlays and prefixed_name in overlays and parser.found_scripts:
# Look up the chrome URL for the overlay
reversed_chrome_url = chrome.reverse_lookup(err, name)
for script in parser.found_scripts:
# Change the URL to an absolute URL.
script = _make_script_absolute(reversed_chrome_url, script)
# Mark the script as potentially pollutable.
marked_scripts.add(script)
err.save_resource("marked_scripts", marked_scripts)
else:
# For all other files, simply throw it at _process_file.
processed = _process_file(err, xpi_package, name, file_data,
name_lower)
# If the file is processed, it will return True. If the process
# goes badly, it will return False. If the processing is skipped,
# it returns None. We should respect that.
if processed is None:
continue
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and not processed:
testendpoint_langpack.test_unsafe_html(err, name, file_data)
# This aids in creating unit tests.
processed_files += 1
# If there aren't any scripts in the package, just skip the next few bits.
if not scripts:
return processed_files
# Save the list of scripts, along with where to find them and the current
# validation state.
existing_scripts = err.get_resource("scripts")
if not existing_scripts:
existing_scripts = []
existing_scripts.append({
"scripts": scripts,
"package": xpi_package,
"state": err.package_stack[:]
})
err.save_resource("scripts", existing_scripts)
return processed_files
