def test_packed_packages(err, xpi_package=None): "Tests XPI and JAR files for naughty content." processed_files = 0 pretested_files = err.get_resource("pretested_files") or [] scripts = set() chrome = err.get_resource("chrome.manifest_nopush") overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource("marked_scripts") if not marked_scripts: marked_scripts = set() # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if ("__MACOSX" in name or name.split("/")[-1].startswith(".")): err.warning( err_id=("testcases_content", "test_packed_packages", "hidden_files"), warning="Hidden files and folders flagged", description="Hidden files and folders difficult the review " "process and can contain sensitive information " "about the system that generated the XPI. Please " "modify the packaging process so that these files " "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=("testcases_content", "test_packaged_packages", "flagged_files"), warning="Flagged filename found", description="Files were found that are either unnecessary " "or have been included unintentionally. They " "should be removed.", filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u"" try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover pass if not err.for_appversions: hash = hashlib.sha1(file_data).hexdigest() if hash in hash_blacklist: err.notice( err_id=("testcases_content", "test_packed_packages", "blacklisted_js_library"), notice="JS Library Detected", description=["JavaScript libraries are discouraged for " "simple add-ons, but are generally " "accepted.", "File %r is a known JS library" % name], filename=name) continue # Process the file. processed = False name_lower = name.lower() if name_lower.endswith((".js", ".jsm")): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith((".xul", ".xml", ".html", ".xhtml", ".xbl")): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)["extension"]) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith("/") else "/%s" % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource("marked_scripts", marked_scripts) else: # For all other files, simply throw it at _process_file. processed = _process_file(err, xpi_package, name, file_data, name_lower) # If the file is processed, it will return True. If the process # goes badly, it will return False. If the processing is skipped, # it returns None. We should respect that. if processed is None: continue # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource("scripts") if not existing_scripts: existing_scripts = [] existing_scripts.append({"scripts": scripts, "package": xpi_package, "state": err.package_stack[:]}) err.save_resource("scripts", existing_scripts) return processed_files
def test_packed_packages(err, xpi_package=None): 'Tests XPI and JAR files for naughty content.' processed_files = 0 pretested_files = err.get_resource('pretested_files') or [] scripts = set() chrome = err.get_resource('chrome.manifest_nopush') overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource('marked_scripts') if not marked_scripts: marked_scripts = set() identified_files = err.metadata.setdefault('identified_files', {}) # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if '__MACOSX' in name or name.split('/')[-1].startswith('.'): err.warning( err_id=('testcases_content', 'test_packed_packages', 'hidden_files'), warning='Hidden files and folders flagged', description='Hidden files and folders complicate the review ' 'process and can contain sensitive information ' 'about the system that generated the XPI. Please ' 'modify the packaging process so that these files ' "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=('testcases_content', 'test_packaged_packages', 'flagged_files'), warning='Flagged filename found', description='Files were found that are either unnecessary ' 'or have been included unintentionally. They ' 'should be removed.', filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u'' try: file_data = xpi_package.read(name) except KeyError: pass except BadZipfile: err.error(('testcases_content', 'test_packed_packages', 'jar_subpackage_corrupt'), 'Package corrupt', 'The package appears to be corrupt.', file=xpi_package.filename) break if not err.for_appversions: hash = hashlib.sha256(file_data).hexdigest() identified = hash_library.get(hash) if identified is not None: identified_files[name] = {'path': identified} err.notice( err_id=('testcases_content', 'test_packed_packages', 'blacklisted_js_library'), notice='JS Library Detected', description=('JavaScript libraries are discouraged for ' 'simple add-ons, but are generally ' 'accepted.', 'File %r is a known JS library' % name), filename=name) continue # Process the file. name_lower = name.lower() if name_lower.endswith(('.js', '.jsm')): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith(('.xul', '.xml', '.html', '.xhtml', '.xbl')): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)['extension']) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith('/') else '/%s' % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) if script: # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource('marked_scripts', marked_scripts) else: err.warning(err_id=('testcases_content', 'test_packed_packages', 'invalid_chrome_url'), warning='Invalid chrome URL', description='The referenced chrome: URL ' 'could not be resolved to a ' 'script file.', filename=name) else: # For all other files, simply throw it at _process_file. _process_file(err, xpi_package, name, file_data, name_lower) # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and name_lower.endswith( ('.html', '.xhtml', '.xul', '.xml', '.xbl', '.properties', '.dtd')): testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource('scripts') if not existing_scripts: existing_scripts = [] existing_scripts.append({ 'scripts': scripts, 'package': xpi_package, 'state': err.package_stack[:] }) err.save_resource('scripts', existing_scripts) return processed_files
def test_packed_packages(err, xpi_package=None): 'Tests XPI and JAR files for naughty content.' processed_files = 0 pretested_files = err.get_resource('pretested_files') or [] scripts = set() chrome = err.get_resource('chrome.manifest_nopush') overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource('marked_scripts') if not marked_scripts: marked_scripts = set() identified_files = err.metadata.setdefault('identified_files', {}) # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if '__MACOSX' in name or name.split('/')[-1].startswith('.'): err.warning( err_id=('testcases_content', 'test_packed_packages', 'hidden_files'), warning='Hidden files and folders flagged', description='Hidden files and folders complicate the review ' 'process and can contain sensitive information ' 'about the system that generated the XPI. Please ' 'modify the packaging process so that these files ' "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=('testcases_content', 'test_packaged_packages', 'flagged_files'), warning='Flagged filename found', description='Files were found that are either unnecessary ' 'or have been included unintentionally. They ' 'should be removed.', filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u'' try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover pass if not err.for_appversions: hash = hashlib.sha256(file_data).hexdigest() identified = hash_library.get(hash) if identified is not None: identified_files[name] = {'path': identified} err.notice( err_id=('testcases_content', 'test_packed_packages', 'blacklisted_js_library'), notice='JS Library Detected', description=('JavaScript libraries are discouraged for ' 'simple add-ons, but are generally ' 'accepted.', 'File %r is a known JS library' % name), filename=name) continue # Process the file. processed = False name_lower = name.lower() if name_lower.endswith(('.js', '.jsm')): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith(('.xul', '.xml', '.html', '.xhtml', '.xbl')): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)['extension']) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith('/') else '/%s' % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) if script: # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource('marked_scripts', marked_scripts) else: err.warning( err_id=('testcases_content', 'test_packed_packages', 'invalid_chrome_url'), warning='Invalid chrome URL', description='The referenced chrome: URL ' 'could not be resolved to a ' 'script file.', filename=name) else: # For all other files, simply throw it at _process_file. processed = _process_file(err, xpi_package, name, file_data, name_lower) # If the file is processed, it will return True. If the process # goes badly, it will return False. If the processing is skipped, # it returns None. We should respect that. if processed is None: continue # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource('scripts') if not existing_scripts: existing_scripts = [] existing_scripts.append({'scripts': scripts, 'package': xpi_package, 'state': err.package_stack[:]}) err.save_resource('scripts', existing_scripts) return processed_files
def test_unsafe_html(): "Tests for unsafe HTML in obstract files." err = ErrorBundle(None, True) langpack.test_unsafe_html(err, None, """ This is an <b>innocent</b> file. Nothing to <a href="#anchor">suspect</a> here. <img src="chrome://asdf/locale/asdf" /> <tag href="#" />""") langpack.test_unsafe_html(err, None, "<tag href='foo' />") langpack.test_unsafe_html(err, None, "<tag src='foo' />") langpack.test_unsafe_html(err, None, "<tag src='/foo/bar' />") assert not err.failed() langpack.test_unsafe_html(err, "asdf", """ This is not an <script>innocent</script> file.""") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html(err, "asdf", """ Nothing to <a href="http://foo.bar/">suspect</a> here.""") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html(err, "asdf", "src='data:foobar") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html(err, "asdf", "src='//remote/resource") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html(err, "asdf", 'href="ftp://foo.bar/') assert err.failed()
def test_unsafe_html(): 'Tests for unsafe HTML in obstract files.' err = ErrorBundle(None, True) langpack.test_unsafe_html( err, None, """ This is an <b>innocent</b> file. Nothing to <a href="#anchor">suspect</a> here. <img src="chrome://asdf/locale/asdf" /> <tag href="#" />""") langpack.test_unsafe_html(err, None, "<tag href='foo' />") langpack.test_unsafe_html(err, None, "<tag src='foo' />") langpack.test_unsafe_html(err, None, "<tag src='/foo/bar' />") assert not err.failed() langpack.test_unsafe_html( err, 'asdf', """ This is not an <script>innocent</script> file.""") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html( err, 'asdf', """ Nothing to <a href="http://foo.bar/">suspect</a> here.""") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html(err, 'asdf', "src='data:foobar") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html(err, 'asdf', "src='//remote/resource") assert err.failed() err = ErrorBundle() langpack.test_unsafe_html(err, 'asdf', 'href="ftp://foo.bar/') assert err.failed()
def test_packed_packages(err, package_contents=None, xpi_package=None): "Tests XPI and JAR files for naughty content." processed_files = 0 hash_whitelist = [x[:-1] for x in open(os.path.join(os.path.dirname(__file__), 'whitelist_hashes.txt')).readlines()] # Iterate each item in the package. for name, data in package_contents.items(): if name.startswith("__MACOSX") or \ name.startswith(".DS_Store"): continue if name.split("/")[-1].startswith("._"): err.notice(("testcases_content", "test_packed_packages", "macintosh_junk"), "Garbage file found.", ["""A junk file has been detected. It may cause problems with proper operation of the add-on down the road.""", "It is recommended that you delete the file"], name) try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover _read_error(err, name) # Skip over whitelisted hashes hash = hashlib.sha1(file_data).hexdigest() if hash in hash_whitelist: continue processed = False # If that item is a container file, unzip it and scan it. if data["extension"] == "jar": # This is either a subpackage or a nested theme. # Whether this is a subpackage or a nested theme is # determined by whether it is in the root folder or not. # Subpackages are always found in a directory such as # /chrome or /content. is_subpackage = name.count("/") > 0 # Unpack the package and load it up. package = StringIO(file_data) sub_xpi = XPIManager(package, name, is_subpackage) if not sub_xpi.zf: err.error(("testcases_content", "test_packed_packages", "jar_subpackage_corrupt"), "Subpackage corrupt.", """The subpackage could not be opened due to issues with corruption. Ensure that the file is valid.""", name) continue temp_contents = sub_xpi.get_file_data() # Let the error bunder know we're in a sub-package. err.push_state(data["name_lower"]) err.set_type(PACKAGE_SUBPACKAGE) # Subpackage testendpoint_validator.test_inner_package(err, temp_contents, sub_xpi) err.tier = 2 package.close() err.pop_state() elif data["extension"] == "xpi": # It's not a subpackage, it's a nested extension. These are # found in multi-extension packages. # Unpack! package = StringIO(file_data) err.push_state(data["name_lower"]) # There are no expected types for packages within a multi- # item package. testendpoint_validator.test_package(err, package, name) err.tier = 2 # Reset to the current tier package.close() err.pop_state() elif data["extension"] in ("xul", "xml", "html", "xhtml"): parser = testendpoint_markup.MarkupParser(err) parser.process(name, charsethelper.decode(file_data), data["extension"]) processed = True elif data["extension"] in ("css", "js", "jsm"): if not file_data: continue file_data = charsethelper.decode(file_data) if data["extension"] == "css": testendpoint_css.test_css_file(err, name, file_data) elif data["extension"] in ("js", "jsm"): testendpoint_js.test_js_file(err, name, file_data) # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 return processed_files
def test_packed_packages(err, xpi_package=None): "Tests XPI and JAR files for naughty content." processed_files = 0 pretested_files = err.get_resource("pretested_files") or [] scripts = set() chrome = err.get_resource("chrome.manifest_nopush") overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource("marked_scripts") if not marked_scripts: marked_scripts = set() identified_files = err.metadata.setdefault("identified_files", {}) # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if ("__MACOSX" in name or name.split("/")[-1].startswith(".")): err.warning( err_id=("testcases_content", "test_packed_packages", "hidden_files"), warning="Hidden files and folders flagged", description="Hidden files and folders complicate the review " "process and can contain sensitive information " "about the system that generated the XPI. Please " "modify the packaging process so that these files " "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=("testcases_content", "test_packaged_packages", "flagged_files"), warning="Flagged filename found", description="Files were found that are either unnecessary " "or have been included unintentionally. They " "should be removed.", filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u"" try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover pass if not err.for_appversions: hash = hashlib.sha256(file_data).hexdigest() identified = hash_library.get(hash) if identified is not None: identified_files[name] = {"path": identified} err.notice(err_id=("testcases_content", "test_packed_packages", "blacklisted_js_library"), notice="JS Library Detected", description=[ "JavaScript libraries are discouraged for " "simple add-ons, but are generally " "accepted.", "File %r is a known JS library" % name ], filename=name) continue # Process the file. processed = False name_lower = name.lower() if name_lower.endswith((".js", ".jsm")): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith((".xul", ".xml", ".html", ".xhtml", ".xbl")): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)["extension"]) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith("/") else "/%s" % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource("marked_scripts", marked_scripts) else: # For all other files, simply throw it at _process_file. processed = _process_file(err, xpi_package, name, file_data, name_lower) # If the file is processed, it will return True. If the process # goes badly, it will return False. If the processing is skipped, # it returns None. We should respect that. if processed is None: continue # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource("scripts") if not existing_scripts: existing_scripts = [] existing_scripts.append({ "scripts": scripts, "package": xpi_package, "state": err.package_stack[:] }) err.save_resource("scripts", existing_scripts) return processed_files