def test_packed_scripts(err, xpi_package): """ Scripts must be tested separately from normal files to allow for markup files to mark scripts as being potentially polluting. """ # This test doesn't apply to subpackages. We keep references to the # subpackage bundles so we can process everything at once in an unpushed # state. if err.is_nested_package(): return scripts = err.get_resource("scripts") if not scripts: return # Get the chrome manifest in case there's information about pollution # exemptions. chrome = err.get_resource("chrome.manifest_nopush") marked_scripts = err.get_resource("marked_scripts") if not marked_scripts: marked_scripts = set() # Process all of the scripts that were found seperately from the rest of # the package contents. for script_bundle in scripts: package = script_bundle["package"] # Set the error bundle's package state to what it was when we first # encountered the script file during the content tests. err.package_stack = script_bundle["state"] for script in script_bundle["scripts"]: file_data = unicodehelper.decode(package.read(script)) if marked_scripts: reversed_script = chrome.reverse_lookup(script_bundle["state"], script) # Run the standard script tests on the script, but mark the # script as pollutable if its chrome URL is marked as being so. testendpoint_js.test_js_file( err, script, file_data, pollutable=reversed_script in marked_scripts) else: # Run the standard script tests on the scripts. testendpoint_js.test_js_file(err, script, file_data) run_regex_tests(file_data, err, script, is_js=True) # We only run this testcase if the package stack is empty, return it to its # original state. err.package_stack = []
def test_packed_packages(err, xpi_package=None): 'Tests XPI and JAR files for naughty content.' processed_files = 0 pretested_files = err.get_resource('pretested_files') or [] scripts = set() chrome = err.get_resource('chrome.manifest_nopush') overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource('marked_scripts') if not marked_scripts: marked_scripts = set() identified_files = err.metadata.setdefault('identified_files', {}) # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if '__MACOSX' in name or name.split('/')[-1].startswith('.'): err.warning( err_id=('testcases_content', 'test_packed_packages', 'hidden_files'), warning='Hidden files and folders flagged', description='Hidden files and folders complicate the review ' 'process and can contain sensitive information ' 'about the system that generated the XPI. Please ' 'modify the packaging process so that these files ' "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=('testcases_content', 'test_packaged_packages', 'flagged_files'), warning='Flagged filename found', description='Files were found that are either unnecessary ' 'or have been included unintentionally. They ' 'should be removed.', filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u'' try: file_data = xpi_package.read(name) except KeyError: pass except BadZipfile: err.error(('testcases_content', 'test_packed_packages', 'jar_subpackage_corrupt'), 'Package corrupt', 'The package appears to be corrupt.', file=xpi_package.filename) break if not err.for_appversions: hash = hashlib.sha256(file_data).hexdigest() identified = hash_library.get(hash) if identified is not None: identified_files[name] = {'path': identified} err.notice( err_id=('testcases_content', 'test_packed_packages', 'blacklisted_js_library'), notice='JS Library Detected', description=('JavaScript libraries are discouraged for ' 'simple add-ons, but are generally ' 'accepted.', 'File %r is a known JS library' % name), filename=name) continue # Process the file. name_lower = name.lower() if name_lower.endswith(('.js', '.jsm')): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith(('.xul', '.xml', '.html', '.xhtml', '.xbl')): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)['extension']) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith('/') else '/%s' % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) if script: # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource('marked_scripts', marked_scripts) else: err.warning(err_id=('testcases_content', 'test_packed_packages', 'invalid_chrome_url'), warning='Invalid chrome URL', description='The referenced chrome: URL ' 'could not be resolved to a ' 'script file.', filename=name) else: # For all other files, simply throw it at _process_file. _process_file(err, xpi_package, name, file_data, name_lower) # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and name_lower.endswith( ('.html', '.xhtml', '.xul', '.xml', '.xbl', '.properties', '.dtd')): testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource('scripts') if not existing_scripts: existing_scripts = [] existing_scripts.append({ 'scripts': scripts, 'package': xpi_package, 'state': err.package_stack[:] }) err.save_resource('scripts', existing_scripts) return processed_files
def _process_file(err, xpi_package, name, file_data, name_lower, pollutable=False): """Process a single file's content tests.""" extension = os.path.splitext(name_lower)[1] # If that item is a container file, unzip it and scan it. if extension == '.jar': # This is either a subpackage or a nested theme. is_subpackage = not err.get_resource('is_multipackage') # Unpack the package and load it up. package = StringIO(file_data) try: sub_xpi = XPIManager(package, mode='r', name=name, subpackage=is_subpackage) except BadZipfile: err.error(('testcases_content', 'test_packed_packages', 'jar_subpackage_corrupt'), 'Subpackage corrupt.', 'The subpackage appears to be corrupt, and could not ' 'be opened.', name) return # Let the error bunder know we're in a sub-package. err.push_state(name) err.detected_type = (PACKAGE_SUBPACKAGE if is_subpackage else PACKAGE_THEME) err.set_tier(1) supported_versions = (err.supported_versions.copy() if err.supported_versions else err.supported_versions) if is_subpackage: testendpoint_validator.test_inner_package(err, sub_xpi) else: testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) err.supported_versions = supported_versions elif extension == '.xpi': # It's not a subpackage, it's a nested extension. These are # found in multi-extension packages. # Unpack! package = StringIO(file_data) err.push_state(name_lower) err.set_tier(1) # There are no expected types for packages within a multi- # item package. testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) # Reset to the current tier else: if not file_data: return # Convert the file data to unicode. file_data = unicodehelper.decode(file_data) if extension in ('.js', '.jsm'): testendpoint_js.test_js_file(err, name, file_data, pollutable=pollutable) elif extension == '.css': testendpoint_css.test_css_file(err, name, file_data) run_regex_tests(file_data, err, filename=name)
def test_packed_scripts(err, xpi_package): """ Scripts must be tested separately from normal files to allow for markup files to mark scripts as being potentially polluting. """ # This test doesn't apply to subpackages. We keep references to the # subpackage bundles so we can process everything at once in an unpushed # state. if err.is_nested_package: return scripts = err.get_resource('scripts') if not scripts: return total_scripts = sum(len(bundle['scripts']) for bundle in scripts) exhaustive = True if total_scripts > MAX_JS_THRESHOLD: err.warning( err_id=('testcases_content', 'packed_js', 'too_much_js'), warning='TOO MUCH JS FOR EXHAUSTIVE VALIDATION', description='There are too many JS files for the validator to ' 'process sequentially. An editor must manually ' 'review the JS in this add-on.') exhaustive = False # Get the chrome manifest in case there's information about pollution # exemptions. chrome = err.get_resource('chrome.manifest_nopush') marked_scripts = err.get_resource('marked_scripts') if not marked_scripts: marked_scripts = set() # Process all of the scripts that were found seperately from the rest of # the package contents. for script_bundle in scripts: package = script_bundle['package'] # Set the error bundle's package state to what it was when we first # encountered the script file during the content tests. for archive in script_bundle['state']: err.push_state(archive) for script in script_bundle['scripts']: file_data = unicodehelper.decode(package.read(script)) run_regex_tests(file_data, err, script) # If we're not running an exhaustive set of tests, skip the full JS # parse and traversal. if not exhaustive: continue if marked_scripts: reversed_script = chrome.reverse_lookup( script_bundle['state'], script) # Run the standard script tests on the script, but mark the # script as pollutable if its chrome URL is marked as being so. testendpoint_js.test_js_file(err, script, file_data, pollutable=reversed_script in marked_scripts) else: # Run the standard script tests on the scripts. testendpoint_js.test_js_file(err, script, file_data) for i in range(len(script_bundle['state'])): err.pop_state()
def test_packed_packages(err, xpi_package=None): "Tests XPI and JAR files for naughty content." processed_files = 0 pretested_files = err.get_resource("pretested_files") or [] scripts = set() chrome = err.get_resource("chrome.manifest_nopush") overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource("marked_scripts") if not marked_scripts: marked_scripts = set() # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if ("__MACOSX" in name or name.split("/")[-1].startswith(".")): err.warning( err_id=("testcases_content", "test_packed_packages", "hidden_files"), warning="Hidden files and folders flagged", description="Hidden files and folders difficult the review " "process and can contain sensitive information " "about the system that generated the XPI. Please " "modify the packaging process so that these files " "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=("testcases_content", "test_packaged_packages", "flagged_files"), warning="Flagged filename found", description="Files were found that are either unnecessary " "or have been included unintentionally. They " "should be removed.", filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u"" try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover pass if not err.for_appversions: hash = hashlib.sha1(file_data).hexdigest() if hash in hash_blacklist: err.notice( err_id=("testcases_content", "test_packed_packages", "blacklisted_js_library"), notice="JS Library Detected", description=["JavaScript libraries are discouraged for " "simple add-ons, but are generally " "accepted.", "File %r is a known JS library" % name], filename=name) continue # Process the file. processed = False name_lower = name.lower() if name_lower.endswith((".js", ".jsm")): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith((".xul", ".xml", ".html", ".xhtml", ".xbl")): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)["extension"]) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith("/") else "/%s" % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource("marked_scripts", marked_scripts) else: # For all other files, simply throw it at _process_file. processed = _process_file(err, xpi_package, name, file_data, name_lower) # If the file is processed, it will return True. If the process # goes badly, it will return False. If the processing is skipped, # it returns None. We should respect that. if processed is None: continue # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource("scripts") if not existing_scripts: existing_scripts = [] existing_scripts.append({"scripts": scripts, "package": xpi_package, "state": err.package_stack[:]}) err.save_resource("scripts", existing_scripts) return processed_files
def _process_file(err, xpi_package, name, file_data, name_lower, pollutable=False): """Process a single file's content tests.""" # If that item is a container file, unzip it and scan it. if name_lower.endswith(".jar"): # This is either a subpackage or a nested theme. is_subpackage = not err.get_resource("is_multipackage") # Unpack the package and load it up. package = StringIO(file_data) try: sub_xpi = XPIManager(package, mode="r", name=name, subpackage=is_subpackage) except Exception: err.error(("testcases_content", "test_packed_packages", "jar_subpackage_corrupt"), "Subpackage corrupt.", "The subpackage could not be opened due to issues " "with corruption. Ensure that the file is valid.", name) return None # Let the error bunder know we're in a sub-package. err.push_state(name) err.detected_type = (PACKAGE_SUBPACKAGE if is_subpackage else PACKAGE_THEME) err.set_tier(1) supported_versions = (err.supported_versions.copy() if err.supported_versions else err.supported_versions) if is_subpackage: testendpoint_validator.test_inner_package(err, sub_xpi) else: testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) err.supported_versions = supported_versions elif name_lower.endswith(".xpi"): # It's not a subpackage, it's a nested extension. These are # found in multi-extension packages. # Unpack! package = StringIO(file_data) err.push_state(name_lower) err.set_tier(1) # There are no expected types for packages within a multi- # item package. testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) # Reset to the current tier elif name_lower.endswith((".css", ".js", ".jsm")): if not file_data: return None # Convert the file data to unicode file_data = unicodehelper.decode(file_data) is_js = False if name_lower.endswith(".css"): testendpoint_css.test_css_file(err, name, file_data) elif name_lower.endswith((".js", ".jsm")): is_js = True testendpoint_js.test_js_file(err, name, file_data, pollutable=pollutable) run_regex_tests(file_data, err, name, is_js=is_js) return True return False
def test_packed_scripts(err, xpi_package): """ Scripts must be tested separately from normal files to allow for markup files to mark scripts as being potentially polluting. """ # This test doesn't apply to subpackages. We keep references to the # subpackage bundles so we can process everything at once in an unpushed # state. if err.is_nested_package: return scripts = err.get_resource("scripts") if not scripts: return total_scripts = sum(len(bundle["scripts"]) for bundle in scripts) exhaustive = True if total_scripts > MAX_JS_THRESHOLD: err.warning( err_id=("testcases_content", "packed_js", "too_much_js"), warning="TOO MUCH JS FOR EXHAUSTIVE VALIDATION", description="There are too many JS files for the validator to " "process sequentially. An editor must manually " "review the JS in this add-on.") exhaustive = False # Get the chrome manifest in case there's information about pollution # exemptions. chrome = err.get_resource("chrome.manifest_nopush") marked_scripts = err.get_resource("marked_scripts") if not marked_scripts: marked_scripts = set() # Process all of the scripts that were found seperately from the rest of # the package contents. for script_bundle in scripts: package = script_bundle["package"] # Set the error bundle's package state to what it was when we first # encountered the script file during the content tests. for archive in script_bundle["state"]: err.push_state(archive) for script in script_bundle["scripts"]: file_data = unicodehelper.decode(package.read(script)) run_regex_tests(file_data, err, script, is_js=True) # If we're not running an exhaustive set of tests, skip the full JS # parse and traversal. if not exhaustive: continue if marked_scripts: reversed_script = chrome.reverse_lookup(script_bundle["state"], script) # Run the standard script tests on the script, but mark the # script as pollutable if its chrome URL is marked as being so. testendpoint_js.test_js_file( err, script, file_data, pollutable=reversed_script in marked_scripts) else: # Run the standard script tests on the scripts. testendpoint_js.test_js_file(err, script, file_data) for i in range(len(script_bundle["state"])): err.pop_state()
def test_packed_packages(err, xpi_package=None): 'Tests XPI and JAR files for naughty content.' processed_files = 0 pretested_files = err.get_resource('pretested_files') or [] scripts = set() chrome = err.get_resource('chrome.manifest_nopush') overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource('marked_scripts') if not marked_scripts: marked_scripts = set() identified_files = err.metadata.setdefault('identified_files', {}) # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if '__MACOSX' in name or name.split('/')[-1].startswith('.'): err.warning( err_id=('testcases_content', 'test_packed_packages', 'hidden_files'), warning='Hidden files and folders flagged', description='Hidden files and folders complicate the review ' 'process and can contain sensitive information ' 'about the system that generated the XPI. Please ' 'modify the packaging process so that these files ' "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=('testcases_content', 'test_packaged_packages', 'flagged_files'), warning='Flagged filename found', description='Files were found that are either unnecessary ' 'or have been included unintentionally. They ' 'should be removed.', filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u'' try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover pass if not err.for_appversions: hash = hashlib.sha256(file_data).hexdigest() identified = hash_library.get(hash) if identified is not None: identified_files[name] = {'path': identified} err.notice( err_id=('testcases_content', 'test_packed_packages', 'blacklisted_js_library'), notice='JS Library Detected', description=('JavaScript libraries are discouraged for ' 'simple add-ons, but are generally ' 'accepted.', 'File %r is a known JS library' % name), filename=name) continue # Process the file. processed = False name_lower = name.lower() if name_lower.endswith(('.js', '.jsm')): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith(('.xul', '.xml', '.html', '.xhtml', '.xbl')): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)['extension']) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith('/') else '/%s' % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) if script: # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource('marked_scripts', marked_scripts) else: err.warning( err_id=('testcases_content', 'test_packed_packages', 'invalid_chrome_url'), warning='Invalid chrome URL', description='The referenced chrome: URL ' 'could not be resolved to a ' 'script file.', filename=name) else: # For all other files, simply throw it at _process_file. processed = _process_file(err, xpi_package, name, file_data, name_lower) # If the file is processed, it will return True. If the process # goes badly, it will return False. If the processing is skipped, # it returns None. We should respect that. if processed is None: continue # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource('scripts') if not existing_scripts: existing_scripts = [] existing_scripts.append({'scripts': scripts, 'package': xpi_package, 'state': err.package_stack[:]}) err.save_resource('scripts', existing_scripts) return processed_files
def _process_file(err, xpi_package, name, file_data, name_lower, pollutable=False): """Process a single file's content tests.""" # If that item is a container file, unzip it and scan it. if name_lower.endswith('.jar'): # This is either a subpackage or a nested theme. is_subpackage = not err.get_resource('is_multipackage') # Unpack the package and load it up. package = StringIO(file_data) try: sub_xpi = XPIManager(package, mode='r', name=name, subpackage=is_subpackage) except BadZipfile: err.error(('testcases_content', 'test_packed_packages', 'jar_subpackage_corrupt'), 'Subpackage corrupt.', 'The subpackage appears to be corrupt, and could not ' 'be opened.', name) return None # Let the error bunder know we're in a sub-package. err.push_state(name) err.detected_type = (PACKAGE_SUBPACKAGE if is_subpackage else PACKAGE_THEME) err.set_tier(1) supported_versions = (err.supported_versions.copy() if err.supported_versions else err.supported_versions) if is_subpackage: testendpoint_validator.test_inner_package(err, sub_xpi) else: testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) err.supported_versions = supported_versions elif name_lower.endswith('.xpi'): # It's not a subpackage, it's a nested extension. These are # found in multi-extension packages. # Unpack! package = StringIO(file_data) err.push_state(name_lower) err.set_tier(1) # There are no expected types for packages within a multi- # item package. testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) # Reset to the current tier elif name_lower.endswith(('.css', '.js', '.jsm')): if not file_data: return None # Convert the file data to unicode file_data = unicodehelper.decode(file_data) is_js = name_lower.endswith(('.js', '.jsm')) if name_lower.endswith('.css'): testendpoint_css.test_css_file(err, name, file_data) elif is_js: testendpoint_js.test_js_file(err, name, file_data, pollutable=pollutable) run_regex_tests(file_data, err, name, is_js=is_js) return True else: if file_data: file_data = unicodehelper.decode(file_data) run_regex_tests(file_data, err, name, explicit=True) return False
def test_packed_packages(err, xpi_package=None): "Tests XPI and JAR files for naughty content." processed_files = 0 pretested_files = err.get_resource("pretested_files") or [] scripts = set() chrome = err.get_resource("chrome.manifest_nopush") overlays = chrome.get_applicable_overlays(err) if chrome else set() marked_scripts = err.get_resource("marked_scripts") if not marked_scripts: marked_scripts = set() identified_files = err.metadata.setdefault("identified_files", {}) # Iterate each item in the package. for name in xpi_package: # Warn for things like __MACOSX directories and .old files. if ("__MACOSX" in name or name.split("/")[-1].startswith(".")): err.warning( err_id=("testcases_content", "test_packed_packages", "hidden_files"), warning="Hidden files and folders flagged", description="Hidden files and folders complicate the review " "process and can contain sensitive information " "about the system that generated the XPI. Please " "modify the packaging process so that these files " "aren't included.", filename=name) continue elif (any(name.endswith(ext) for ext in FLAGGED_EXTENSIONS) or name in FLAGGED_FILES): err.warning( err_id=("testcases_content", "test_packaged_packages", "flagged_files"), warning="Flagged filename found", description="Files were found that are either unnecessary " "or have been included unintentionally. They " "should be removed.", filename=name) continue # Skip the file if it's in the pre-tested files resource. This skips # things like Jetpack files. if name in pretested_files: continue # Read the file from the archive if possible. file_data = u"" try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover pass if not err.for_appversions: hash = hashlib.sha256(file_data).hexdigest() identified = hash_library.get(hash) if identified is not None: identified_files[name] = {"path": identified} err.notice(err_id=("testcases_content", "test_packed_packages", "blacklisted_js_library"), notice="JS Library Detected", description=[ "JavaScript libraries are discouraged for " "simple add-ons, but are generally " "accepted.", "File %r is a known JS library" % name ], filename=name) continue # Process the file. processed = False name_lower = name.lower() if name_lower.endswith((".js", ".jsm")): # Add the scripts to a list to be processed later. scripts.add(name) elif name_lower.endswith((".xul", ".xml", ".html", ".xhtml", ".xbl")): # Process markup files outside of _process_file so we can get # access to information about linked scripts and such. parser = testendpoint_markup.MarkupParser(err) parser.process(name, file_data, xpi_package.info(name)["extension"]) run_regex_tests(file_data, err, name) # Make sure the name is prefixed with a forward slash. prefixed_name = name if name.startswith("/") else "/%s" % name # Mark scripts as pollutable if this is an overlay file and there # are scripts to mark. if overlays and prefixed_name in overlays and parser.found_scripts: # Look up the chrome URL for the overlay reversed_chrome_url = chrome.reverse_lookup(err, name) for script in parser.found_scripts: # Change the URL to an absolute URL. script = _make_script_absolute(reversed_chrome_url, script) # Mark the script as potentially pollutable. marked_scripts.add(script) err.save_resource("marked_scripts", marked_scripts) else: # For all other files, simply throw it at _process_file. processed = _process_file(err, xpi_package, name, file_data, name_lower) # If the file is processed, it will return True. If the process # goes badly, it will return False. If the processing is skipped, # it returns None. We should respect that. if processed is None: continue # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 # If there aren't any scripts in the package, just skip the next few bits. if not scripts: return processed_files # Save the list of scripts, along with where to find them and the current # validation state. existing_scripts = err.get_resource("scripts") if not existing_scripts: existing_scripts = [] existing_scripts.append({ "scripts": scripts, "package": xpi_package, "state": err.package_stack[:] }) err.save_resource("scripts", existing_scripts) return processed_files