def test_packed_packages(err, xpi_package=None):
"Tests XPI and JAR files for naughty content."
processed_files = 0
pretested_files = err.get_resource("pretested_files") or []
scripts = set()
chrome = err.get_resource("chrome.manifest_nopush")
overlays = chrome.get_applicable_overlays(err) if chrome else set()
marked_scripts = err.get_resource("marked_scripts")
if not marked_scripts:
marked_scripts = set()
# Iterate each item in the package.
for name in xpi_package:
# Warn for things like __MACOSX directories and .old files.
if ("__MACOSX" in name or
name.split("/")[-1].startswith(".")):
err.warning(
err_id=("testcases_content", "test_packed_packages",
"hidden_files"),
warning="Hidden files and folders flagged",
description="Hidden files and folders difficult the review "
"process and can contain sensitive information "
"about the system that generated the XPI. Please "
"modify the packaging process so that these files "
"aren't included.",
filename=name)
continue
elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or
name in FLAGGED_FILES):
err.warning(
err_id=("testcases_content", "test_packaged_packages",
"flagged_files"),
warning="Flagged filename found",
description="Files were found that are either unnecessary "
"or have been included unintentionally. They "
"should be removed.",
filename=name)
continue
# Skip the file if it's in the pre-tested files resource. This skips
# things like Jetpack files.
if name in pretested_files:
continue
# Read the file from the archive if possible.
file_data = u""
try:
file_data = xpi_package.read(name)
except KeyError: # pragma: no cover
pass
if not err.for_appversions:
hash = hashlib.sha1(file_data).hexdigest()
if hash in hash_blacklist:
err.notice(
err_id=("testcases_content",
"test_packed_packages",
"blacklisted_js_library"),
notice="JS Library Detected",
description=["JavaScript libraries are discouraged for "
"simple add-ons, but are generally "
"accepted.",
"File %r is a known JS library" % name],
filename=name)
continue
# Process the file.
processed = False
name_lower = name.lower()
if name_lower.endswith((".js", ".jsm")):
# Add the scripts to a list to be processed later.
scripts.add(name)
elif name_lower.endswith((".xul", ".xml", ".html", ".xhtml", ".xbl")):
# Process markup files outside of _process_file so we can get
# access to information about linked scripts and such.
parser = testendpoint_markup.MarkupParser(err)
parser.process(name, file_data,
xpi_package.info(name)["extension"])
run_regex_tests(file_data, err, name)
# Make sure the name is prefixed with a forward slash.
prefixed_name = name if name.startswith("/") else "/%s" % name
# Mark scripts as pollutable if this is an overlay file and there
# are scripts to mark.
if overlays and prefixed_name in overlays and parser.found_scripts:
# Look up the chrome URL for the overlay
reversed_chrome_url = chrome.reverse_lookup(err, name)
for script in parser.found_scripts:
# Change the URL to an absolute URL.
script = _make_script_absolute(reversed_chrome_url, script)
# Mark the script as potentially pollutable.
marked_scripts.add(script)
err.save_resource("marked_scripts", marked_scripts)
else:
# For all other files, simply throw it at _process_file.
processed = _process_file(err, xpi_package, name, file_data,
name_lower)
# If the file is processed, it will return True. If the process
# goes badly, it will return False. If the processing is skipped,
# it returns None. We should respect that.
if processed is None:
continue
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and not processed:
testendpoint_langpack.test_unsafe_html(err, name, file_data)
# This aids in creating unit tests.
processed_files += 1
# If there aren't any scripts in the package, just skip the next few bits.
if not scripts:
return processed_files
# Save the list of scripts, along with where to find them and the current
# validation state.
existing_scripts = err.get_resource("scripts")
if not existing_scripts:
existing_scripts = []
existing_scripts.append({"scripts": scripts,
"package": xpi_package,
"state": err.package_stack[:]})
err.save_resource("scripts", existing_scripts)
return processed_files
def test_packed_packages(err, xpi_package=None):
'Tests XPI and JAR files for naughty content.'
processed_files = 0
pretested_files = err.get_resource('pretested_files') or []
scripts = set()
chrome = err.get_resource('chrome.manifest_nopush')
overlays = chrome.get_applicable_overlays(err) if chrome else set()
marked_scripts = err.get_resource('marked_scripts')
if not marked_scripts:
marked_scripts = set()
identified_files = err.metadata.setdefault('identified_files', {})
# Iterate each item in the package.
for name in xpi_package:
# Warn for things like __MACOSX directories and .old files.
if '__MACOSX' in name or name.split('/')[-1].startswith('.'):
err.warning(
err_id=('testcases_content', 'test_packed_packages',
'hidden_files'),
warning='Hidden files and folders flagged',
description='Hidden files and folders complicate the review '
'process and can contain sensitive information '
'about the system that generated the XPI. Please '
'modify the packaging process so that these files '
"aren't included.",
filename=name)
continue
elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS)
or name in FLAGGED_FILES):
err.warning(
err_id=('testcases_content', 'test_packaged_packages',
'flagged_files'),
warning='Flagged filename found',
description='Files were found that are either unnecessary '
'or have been included unintentionally. They '
'should be removed.',
filename=name)
continue
# Skip the file if it's in the pre-tested files resource. This skips
# things like Jetpack files.
if name in pretested_files:
continue
# Read the file from the archive if possible.
file_data = u''
try:
file_data = xpi_package.read(name)
except KeyError:
pass
except BadZipfile:
err.error(('testcases_content', 'test_packed_packages',
'jar_subpackage_corrupt'),
'Package corrupt',
'The package appears to be corrupt.',
file=xpi_package.filename)
break
if not err.for_appversions:
hash = hashlib.sha256(file_data).hexdigest()
identified = hash_library.get(hash)
if identified is not None:
identified_files[name] = {'path': identified}
err.notice(
err_id=('testcases_content', 'test_packed_packages',
'blacklisted_js_library'),
notice='JS Library Detected',
description=('JavaScript libraries are discouraged for '
'simple add-ons, but are generally '
'accepted.',
'File %r is a known JS library' % name),
filename=name)
continue
# Process the file.
name_lower = name.lower()
if name_lower.endswith(('.js', '.jsm')):
# Add the scripts to a list to be processed later.
scripts.add(name)
elif name_lower.endswith(('.xul', '.xml', '.html', '.xhtml', '.xbl')):
# Process markup files outside of _process_file so we can get
# access to information about linked scripts and such.
parser = testendpoint_markup.MarkupParser(err)
parser.process(name, file_data,
xpi_package.info(name)['extension'])
run_regex_tests(file_data, err, name)
# Make sure the name is prefixed with a forward slash.
prefixed_name = name if name.startswith('/') else '/%s' % name
# Mark scripts as pollutable if this is an overlay file and there
# are scripts to mark.
if overlays and prefixed_name in overlays and parser.found_scripts:
# Look up the chrome URL for the overlay
reversed_chrome_url = chrome.reverse_lookup(err, name)
for script in parser.found_scripts:
# Change the URL to an absolute URL.
script = _make_script_absolute(reversed_chrome_url, script)
if script:
# Mark the script as potentially pollutable.
marked_scripts.add(script)
err.save_resource('marked_scripts', marked_scripts)
else:
err.warning(err_id=('testcases_content',
'test_packed_packages',
'invalid_chrome_url'),
warning='Invalid chrome URL',
description='The referenced chrome: URL '
'could not be resolved to a '
'script file.',
filename=name)
else:
# For all other files, simply throw it at _process_file.
_process_file(err, xpi_package, name, file_data, name_lower)
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and name_lower.endswith(
('.html', '.xhtml', '.xul', '.xml', '.xbl', '.properties',
'.dtd')):
testendpoint_langpack.test_unsafe_html(err, name, file_data)
# This aids in creating unit tests.
processed_files += 1
# If there aren't any scripts in the package, just skip the next few bits.
if not scripts:
return processed_files
# Save the list of scripts, along with where to find them and the current
# validation state.
existing_scripts = err.get_resource('scripts')
if not existing_scripts:
existing_scripts = []
existing_scripts.append({
'scripts': scripts,
'package': xpi_package,
'state': err.package_stack[:]
})
err.save_resource('scripts', existing_scripts)
return processed_files
def test_packed_packages(err, xpi_package=None):
'Tests XPI and JAR files for naughty content.'
processed_files = 0
pretested_files = err.get_resource('pretested_files') or []
scripts = set()
chrome = err.get_resource('chrome.manifest_nopush')
overlays = chrome.get_applicable_overlays(err) if chrome else set()
marked_scripts = err.get_resource('marked_scripts')
if not marked_scripts:
marked_scripts = set()
identified_files = err.metadata.setdefault('identified_files', {})
# Iterate each item in the package.
for name in xpi_package:
# Warn for things like __MACOSX directories and .old files.
if '__MACOSX' in name or name.split('/')[-1].startswith('.'):
err.warning(
err_id=('testcases_content', 'test_packed_packages',
'hidden_files'),
warning='Hidden files and folders flagged',
description='Hidden files and folders complicate the review '
'process and can contain sensitive information '
'about the system that generated the XPI. Please '
'modify the packaging process so that these files '
"aren't included.",
filename=name)
continue
elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or
name in FLAGGED_FILES):
err.warning(
err_id=('testcases_content', 'test_packaged_packages',
'flagged_files'),
warning='Flagged filename found',
description='Files were found that are either unnecessary '
'or have been included unintentionally. They '
'should be removed.',
filename=name)
continue
# Skip the file if it's in the pre-tested files resource. This skips
# things like Jetpack files.
if name in pretested_files:
continue
# Read the file from the archive if possible.
file_data = u''
try:
file_data = xpi_package.read(name)
except KeyError: # pragma: no cover
pass
if not err.for_appversions:
hash = hashlib.sha256(file_data).hexdigest()
identified = hash_library.get(hash)
if identified is not None:
identified_files[name] = {'path': identified}
err.notice(
err_id=('testcases_content',
'test_packed_packages',
'blacklisted_js_library'),
notice='JS Library Detected',
description=('JavaScript libraries are discouraged for '
'simple add-ons, but are generally '
'accepted.',
'File %r is a known JS library' % name),
filename=name)
continue
# Process the file.
processed = False
name_lower = name.lower()
if name_lower.endswith(('.js', '.jsm')):
# Add the scripts to a list to be processed later.
scripts.add(name)
elif name_lower.endswith(('.xul', '.xml', '.html', '.xhtml', '.xbl')):
# Process markup files outside of _process_file so we can get
# access to information about linked scripts and such.
parser = testendpoint_markup.MarkupParser(err)
parser.process(name, file_data,
xpi_package.info(name)['extension'])
run_regex_tests(file_data, err, name)
# Make sure the name is prefixed with a forward slash.
prefixed_name = name if name.startswith('/') else '/%s' % name
# Mark scripts as pollutable if this is an overlay file and there
# are scripts to mark.
if overlays and prefixed_name in overlays and parser.found_scripts:
# Look up the chrome URL for the overlay
reversed_chrome_url = chrome.reverse_lookup(err, name)
for script in parser.found_scripts:
# Change the URL to an absolute URL.
script = _make_script_absolute(reversed_chrome_url, script)
if script:
# Mark the script as potentially pollutable.
marked_scripts.add(script)
err.save_resource('marked_scripts', marked_scripts)
else:
err.warning(
err_id=('testcases_content',
'test_packed_packages',
'invalid_chrome_url'),
warning='Invalid chrome URL',
description='The referenced chrome: URL '
'could not be resolved to a '
'script file.',
filename=name)
else:
# For all other files, simply throw it at _process_file.
processed = _process_file(err, xpi_package, name, file_data,
name_lower)
# If the file is processed, it will return True. If the process
# goes badly, it will return False. If the processing is skipped,
# it returns None. We should respect that.
if processed is None:
continue
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and not processed:
testendpoint_langpack.test_unsafe_html(err, name, file_data)
# This aids in creating unit tests.
processed_files += 1
# If there aren't any scripts in the package, just skip the next few bits.
if not scripts:
return processed_files
# Save the list of scripts, along with where to find them and the current
# validation state.
existing_scripts = err.get_resource('scripts')
if not existing_scripts:
existing_scripts = []
existing_scripts.append({'scripts': scripts,
'package': xpi_package,
'state': err.package_stack[:]})
err.save_resource('scripts', existing_scripts)
return processed_files
def test_unsafe_html():
"Tests for unsafe HTML in obstract files."
err = ErrorBundle(None, True)
langpack.test_unsafe_html(err, None, """
This is an <b>innocent</b> file.
Nothing to <a href="#anchor">suspect</a> here.
<img src="chrome://asdf/locale/asdf" />
<tag href="#" />""")
langpack.test_unsafe_html(err, None, "<tag href='foo' />")
langpack.test_unsafe_html(err, None, "<tag src='foo' />")
langpack.test_unsafe_html(err, None, "<tag src='/foo/bar' />")
assert not err.failed()
langpack.test_unsafe_html(err, "asdf", """
This is not an <script>innocent</script> file.""")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(err, "asdf", """
Nothing to <a href="http://foo.bar/">suspect</a> here.""")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(err, "asdf", "src='data:foobar")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(err, "asdf", "src='//remote/resource")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(err, "asdf", 'href="ftp://foo.bar/')
assert err.failed()
def test_unsafe_html():
'Tests for unsafe HTML in obstract files.'
err = ErrorBundle(None, True)
langpack.test_unsafe_html(
err, None, """
This is an <b>innocent</b> file.
Nothing to <a href="#anchor">suspect</a> here.
<img src="chrome://asdf/locale/asdf" />
<tag href="#" />""")
langpack.test_unsafe_html(err, None, "<tag href='foo' />")
langpack.test_unsafe_html(err, None, "<tag src='foo' />")
langpack.test_unsafe_html(err, None, "<tag src='/foo/bar' />")
assert not err.failed()
langpack.test_unsafe_html(
err, 'asdf', """
This is not an <script>innocent</script> file.""")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(
err, 'asdf', """
Nothing to <a href="http://foo.bar/">suspect</a> here.""")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(err, 'asdf', "src='data:foobar")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(err, 'asdf', "src='//remote/resource")
assert err.failed()
err = ErrorBundle()
langpack.test_unsafe_html(err, 'asdf', 'href="ftp://foo.bar/')
assert err.failed()
def test_packed_packages(err, package_contents=None, xpi_package=None):
"Tests XPI and JAR files for naughty content."
processed_files = 0
hash_whitelist = [x[:-1] for x in
open(os.path.join(os.path.dirname(__file__),
'whitelist_hashes.txt')).readlines()]
# Iterate each item in the package.
for name, data in package_contents.items():
if name.startswith("__MACOSX") or \
name.startswith(".DS_Store"):
continue
if name.split("/")[-1].startswith("._"):
err.notice(("testcases_content",
"test_packed_packages",
"macintosh_junk"),
"Garbage file found.",
["""A junk file has been detected. It may cause
problems with proper operation of the add-on down the
road.""",
"It is recommended that you delete the file"],
name)
try:
file_data = xpi_package.read(name)
except KeyError: # pragma: no cover
_read_error(err, name)
# Skip over whitelisted hashes
hash = hashlib.sha1(file_data).hexdigest()
if hash in hash_whitelist:
continue
processed = False
# If that item is a container file, unzip it and scan it.
if data["extension"] == "jar":
# This is either a subpackage or a nested theme.
# Whether this is a subpackage or a nested theme is
# determined by whether it is in the root folder or not.
# Subpackages are always found in a directory such as
# /chrome or /content.
is_subpackage = name.count("/") > 0
# Unpack the package and load it up.
package = StringIO(file_data)
sub_xpi = XPIManager(package, name, is_subpackage)
if not sub_xpi.zf:
err.error(("testcases_content",
"test_packed_packages",
"jar_subpackage_corrupt"),
"Subpackage corrupt.",
"""The subpackage could not be opened due to
issues with corruption. Ensure that the file
is valid.""",
name)
continue
temp_contents = sub_xpi.get_file_data()
# Let the error bunder know we're in a sub-package.
err.push_state(data["name_lower"])
err.set_type(PACKAGE_SUBPACKAGE) # Subpackage
testendpoint_validator.test_inner_package(err,
temp_contents,
sub_xpi)
err.tier = 2
package.close()
err.pop_state()
elif data["extension"] == "xpi":
# It's not a subpackage, it's a nested extension. These are
# found in multi-extension packages.
# Unpack!
package = StringIO(file_data)
err.push_state(data["name_lower"])
# There are no expected types for packages within a multi-
# item package.
testendpoint_validator.test_package(err, package, name)
err.tier = 2 # Reset to the current tier
package.close()
err.pop_state()
elif data["extension"] in ("xul", "xml", "html", "xhtml"):
parser = testendpoint_markup.MarkupParser(err)
parser.process(name,
charsethelper.decode(file_data),
data["extension"])
processed = True
elif data["extension"] in ("css", "js", "jsm"):
if not file_data:
continue
file_data = charsethelper.decode(file_data)
if data["extension"] == "css":
testendpoint_css.test_css_file(err,
name,
file_data)
elif data["extension"] in ("js", "jsm"):
testendpoint_js.test_js_file(err,
name,
file_data)
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and not processed:
testendpoint_langpack.test_unsafe_html(err,
name,
file_data)
# This aids in creating unit tests.
processed_files += 1
return processed_files
def test_packed_packages(err, xpi_package=None):
"Tests XPI and JAR files for naughty content."
processed_files = 0
pretested_files = err.get_resource("pretested_files") or []
scripts = set()
chrome = err.get_resource("chrome.manifest_nopush")
overlays = chrome.get_applicable_overlays(err) if chrome else set()
marked_scripts = err.get_resource("marked_scripts")
if not marked_scripts:
marked_scripts = set()
identified_files = err.metadata.setdefault("identified_files", {})
# Iterate each item in the package.
for name in xpi_package:
# Warn for things like __MACOSX directories and .old files.
if ("__MACOSX" in name or name.split("/")[-1].startswith(".")):
err.warning(
err_id=("testcases_content", "test_packed_packages",
"hidden_files"),
warning="Hidden files and folders flagged",
description="Hidden files and folders complicate the review "
"process and can contain sensitive information "
"about the system that generated the XPI. Please "
"modify the packaging process so that these files "
"aren't included.",
filename=name)
continue
elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS)
or name in FLAGGED_FILES):
err.warning(
err_id=("testcases_content", "test_packaged_packages",
"flagged_files"),
warning="Flagged filename found",
description="Files were found that are either unnecessary "
"or have been included unintentionally. They "
"should be removed.",
filename=name)
continue
# Skip the file if it's in the pre-tested files resource. This skips
# things like Jetpack files.
if name in pretested_files:
continue
# Read the file from the archive if possible.
file_data = u""
try:
file_data = xpi_package.read(name)
except KeyError: # pragma: no cover
pass
if not err.for_appversions:
hash = hashlib.sha256(file_data).hexdigest()
identified = hash_library.get(hash)
if identified is not None:
identified_files[name] = {"path": identified}
err.notice(err_id=("testcases_content", "test_packed_packages",
"blacklisted_js_library"),
notice="JS Library Detected",
description=[
"JavaScript libraries are discouraged for "
"simple add-ons, but are generally "
"accepted.",
"File %r is a known JS library" % name
],
filename=name)
continue
# Process the file.
processed = False
name_lower = name.lower()
if name_lower.endswith((".js", ".jsm")):
# Add the scripts to a list to be processed later.
scripts.add(name)
elif name_lower.endswith((".xul", ".xml", ".html", ".xhtml", ".xbl")):
# Process markup files outside of _process_file so we can get
# access to information about linked scripts and such.
parser = testendpoint_markup.MarkupParser(err)
parser.process(name, file_data,
xpi_package.info(name)["extension"])
run_regex_tests(file_data, err, name)
# Make sure the name is prefixed with a forward slash.
prefixed_name = name if name.startswith("/") else "/%s" % name
# Mark scripts as pollutable if this is an overlay file and there
# are scripts to mark.
if overlays and prefixed_name in overlays and parser.found_scripts:
# Look up the chrome URL for the overlay
reversed_chrome_url = chrome.reverse_lookup(err, name)
for script in parser.found_scripts:
# Change the URL to an absolute URL.
script = _make_script_absolute(reversed_chrome_url, script)
# Mark the script as potentially pollutable.
marked_scripts.add(script)
err.save_resource("marked_scripts", marked_scripts)
else:
# For all other files, simply throw it at _process_file.
processed = _process_file(err, xpi_package, name, file_data,
name_lower)
# If the file is processed, it will return True. If the process
# goes badly, it will return False. If the processing is skipped,
# it returns None. We should respect that.
if processed is None:
continue
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and not processed:
testendpoint_langpack.test_unsafe_html(err, name, file_data)
# This aids in creating unit tests.
processed_files += 1
# If there aren't any scripts in the package, just skip the next few bits.
if not scripts:
return processed_files
# Save the list of scripts, along with where to find them and the current
# validation state.
existing_scripts = err.get_resource("scripts")
if not existing_scripts:
existing_scripts = []
existing_scripts.append({
"scripts": scripts,
"package": xpi_package,
"state": err.package_stack[:]
})
err.save_resource("scripts", existing_scripts)
return processed_files