def _do_test(path, test, failure=True, require_install=False, set_type=0): package_data = open(path, "rb") package = XPIManager(package_data, path) contents = package.get_file_data() err = ErrorBundle() # Populate in the dependencies. if set_type: err.set_type(set_type) # Conduit test requires type if require_install: err.save_resource("has_install_rdf", True) rdf_data = package.read("install.rdf") install_rdf = RDFParser(rdf_data) err.save_resource("install_rdf", install_rdf) test(err, contents, package) print err.print_summary(verbose=True) if failure: assert err.failed() else: assert not err.failed() return err
def _do_test(path, test, failure=True, require_install=False, set_type=0, listed=False, xpi_mode='r'): package_data = open(path, 'rb') package = XPIManager(package_data, mode=xpi_mode, name=path) err = ErrorBundle() if listed: err.save_resource('listed', True) # Populate in the dependencies. if set_type: err.detected_type = set_type # Conduit test requires type if require_install: err.save_resource('has_install_rdf', True) rdf_data = package.read('install.rdf') install_rdf = RDFParser(err, rdf_data) err.save_resource('install_rdf', install_rdf) populate_chrome_manifest(err, package) test(err, package) print err.print_summary(verbose=True) if failure: assert err.failed() else: assert not err.failed() return err
def _do_test(path, test, failure=True, require_install=False, set_type=0, listed=False, xpi_mode="r"): package_data = open(path, "rb") package = XPIManager(package_data, mode=xpi_mode, name=path) err = ErrorBundle() if listed: err.save_resource("listed", True) # Populate in the dependencies. if set_type: err.detected_type = set_type # Conduit test requires type if require_install: err.save_resource("has_install_rdf", True) rdf_data = package.read("install.rdf") install_rdf = RDFParser(err, rdf_data) err.save_resource("install_rdf", install_rdf) populate_chrome_manifest(err, package) test(err, package) print err.print_summary(verbose=True) if failure: assert err.failed() else: assert not err.failed() return err
def test_bad_file(): "Tests that the XPI manager correctly reports a bad XPI file." x = XPIManager("tests/resources/junk.xpi") assert not x.zf x = XPIManager("tests/resources/corrupt.xpi") assert x.test()
def test_get_list(): """Test that the manager can read the file listing.""" z = XPIManager(get_path('xpi/install_rdf_only.xpi')) assert not z.contents_cache assert z.package_contents() assert z.contents_cache # Spelling check! z.contents_cache = 'foo' eq_(z.package_contents(), 'foo')
def test_bad_file(): """Tests that the XPI manager correctly reports a bad XPI file.""" try: x = XPIManager(get_path("junk.xpi")) except BadZipfile: pass x = XPIManager(get_path("corrupt.xpi")) assert x.test()
def test_get_list(): "Test that the manager can read the file listing" z = XPIManager("tests/resources/xpi/install_rdf_only.xpi") assert not z.contents_cache assert z.package_contents() assert z.contents_cache # Spelling check! z.contents_cache = "foo" eq_(z.package_contents(), "foo")
def test_bad_file(): "Tests that the XPI manager correctly reports a bad XPI file." try: x = XPIManager("tests/resources/junk.xpi") except zipfile.BadZipfile: pass x = XPIManager("tests/resources/corrupt.xpi") assert x.test()
def test_write_file(): """Test that a file can be written in UTF-8 to the package.""" with tempfile.NamedTemporaryFile(delete=False) as t: temp_fn = t.name try: z = XPIManager(temp_fn, mode='w') f, d = 'install.rdf', '注目のコレクション'.decode('utf-8') z.write(f, d) eq_(z.read(f), d.encode('utf-8')) finally: os.unlink(temp_fn)
def clean_xpifile(self): "Make sure that the uploaded file is a valid XPI file." xpifile = self.cleaned_data['xpifile'] if xpifile: try: xpi = XPIManager(xpifile, name=xpifile.name) if xpi.test(): raise except: raise forms.ValidationError(_("File doesn't seem to be valid XPI")) return xpifile
def test_blacklisted_files(): """Tests the validator's ability to hash each individual file and (based on this information) determine whether the addon passes or fails the validation process.""" package_data = open("tests/resources/libraryblacklist/blocked.xpi") package = XPIManager(package_data, "blocked.xpi") contents = package.get_file_data() err = ErrorBundle() libblacklist.test_library_blacklist(err, contents, package) print err.print_summary() assert err.notices
def test_package(err, file_, name, expectation=PACKAGE_ANY, for_appversions=None): "Begins tests for the package." # Load up a new instance of an XPI. try: package = XPIManager(file_, mode="r", name=name) except: # Die on this one because the file won't open. return err.error(("main", "test_package", "unopenable"), "The XPI could not be opened.") # Test the XPI file for corruption. if package.test(): return err.error(("main", "test_package", "corrupt"), "XPI package appears to be corrupt.") if package.extension in assumed_extensions: assumed_type = assumed_extensions[package.extension] # Is the user expecting a different package type? if not expectation in (PACKAGE_ANY, assumed_type): err.error(("main", "test_package", "unexpected_type"), "Unexpected package type (found theme)") # Test the install.rdf file to see if we can get the type that way. has_install_rdf = "install.rdf" in package if has_install_rdf: _load_install_rdf(err, package, expectation) try: output = test_inner_package(err, package, for_appversions) except ValidationTimeout as ex: err.error( err_id=("main", "test_package", "timeout"), error="Validation timed out", description=["The validation process took too long to " "complete. Contact an addons.mozilla.org editor " "for more information.", str(ex)]) output = None return output
def _get_locale_manager(err, xpi_package, description, no_cache=False): 'Returns the XPIManager object for a locale' if not description['jarred']: return xpi_package path = description['path'] if path in LOCALE_CACHE and not no_cache: return LOCALE_CACHE[path] if path not in xpi_package: # TODO: Pass the filename of the triple's manifest. err.warning( ('testcases_l10ncompleteness', '_get_locale_manager', 'manager_absent'), 'Listed locale does not exist', [ 'A locale JAR is listed in chrome.manifest, but it could ' 'not be located. Check the spelling and capitalization ' 'of the path.', 'Missing JAR: %s' % path ], filename='chrome.manifest') return None jar = StringIO(xpi_package.read(path)) locale = XPIManager(jar, mode='r', name=path) if not no_cache: LOCALE_CACHE[path] = locale return locale
def _get_locale_manager(err, xpi_package, description, no_cache=False): "Returns the XPIManager object for a locale" if not description["jarred"]: return xpi_package path = description["path"] if path in LOCALE_CACHE and not no_cache: return LOCALE_CACHE[path] if path not in xpi_package: # TODO: Pass the filename of the triple's manifest. err.warning( ("testcases_l10ncompleteness", "_get_locale_manager", "manager_absent"), "Listed locale does not exist", [ "A locale JAR is listed in chrome.manifest, but it could " "not be located. Check the spelling and capitalization " "of the path.", "Missing JAR: %s" % path ], filename="chrome.manifest") return None jar = StringIO(xpi_package.read(path)) locale = XPIManager(jar, mode="r", name=path) if not no_cache: LOCALE_CACHE[path] = locale return locale
def test_blacklisted_files(): """ Tests the validator's ability to hash each individual file and (based on this information) determine whether the addon passes or fails the validation process. """ package_data = open('tests/resources/libraryblacklist/blocked.xpi') package = XPIManager(package_data, mode='r', name='blocked.xpi') err = ErrorBundle() test_content.test_packed_packages(err, package) print err.print_summary() assert err.notices assert not err.failed() eq_( err.metadata.get('identified_files'), { 'test.js': { 'path': 'This file is a false script to facilitate ' 'testing of library blacklisting.' } })
def test_missing_file(): """Tests that the XPI manager correctly reports a missing XPI file.""" passed = False try: x = XPIManager('foo.bar') except: passed = True assert passed
def test_package(err, file_, name, expectation=PACKAGE_ANY, for_appversions=None): "Begins tests for the package." # Load up a new instance of an XPI. try: package = XPIManager(file_, mode="r", name=name) has_package_json = "package.json" in package has_install_rdf = "install.rdf" in package # install.rdf? | package.json? | error | use-file # Yes | No | No | install.rdf # Yes | Yes | No | install.rdf # No | No | Yes | install.rdf # No | Yes | No | package.json if has_package_json: _load_package_json(err, package, expectation) if has_install_rdf: _load_install_rdf(err, package, expectation) except IOError: # Die on this one because the file won't open. return err.error(("main", "test_package", "unopenable"), "The XPI could not be opened.") except (BadZipfile, zlib_error): # Die if the zip file is corrupt. return err.error( ("submain", "_load_install_rdf", "badzipfile"), error="Corrupt ZIP file", description="We were unable to decompress the zip file.") if package.extension in assumed_extensions: assumed_type = assumed_extensions[package.extension] # Is the user expecting a different package type? if not expectation in (PACKAGE_ANY, assumed_type): err.error(("main", "test_package", "unexpected_type"), "Unexpected package type (found theme)") try: output = test_inner_package(err, package, for_appversions) except ValidationTimeout as ex: err.error(err_id=("main", "test_package", "timeout"), error="Validation timed out", description=[ "The validation process took too long to " "complete. Contact an addons.mozilla.org editor " "for more information.", str(ex) ]) output = None return output
def test_mismatched_module_version(): """ Tests that add-ons using modules from a version of the SDK other than the version they claim. """ xpi = XPIManager('tests/resources/jetpack/jetpack-1.8-pretending-1.8.1.xpi') err = _do_test(xpi) assert err.failed() assert any(w['id'][2] == 'mismatched_version' for w in err.warnings)
def test_new_module_location_spec(): """ Tests that we don't fail for missing modules in add-ons generated with newer versions of the SDK. """ xpi = XPIManager("tests/resources/jetpack/jetpack-1.14.xpi") err = _do_test(xpi) assert not any(w["id"][2] == "missing_jetpack_module" for w in err.warnings)
def test_validate_libs_in_compat_mode(): xpi = "tests/resources/libraryblacklist/addon_with_mootools.xpi" with open(xpi) as data: package = XPIManager(data, mode="r", name="addon_with_mootools.xpi") err = ErrorBundle(for_appversions=FX9_DEFINITION) test_content.test_packed_packages(err, package) assert err.get_resource("scripts"), ( "expected mootools scripts to be marked for proessing") eq_( err.get_resource("scripts")[0]["scripts"], set(["content/mootools.js"]))
def test_future_sdkversion(): """ Test that if the developer is using a verison of the SDK that's newer than the latest recognized version, we don't throw an error. """ xpi = XPIManager("tests/resources/jetpack/jetpack-1.8-future.xpi") err = _do_test(xpi, allow_old_sdk=False) print err.print_summary(verbose=True) assert not err.failed()
def _test_type(file_, expectation, failure=False): "Tests a file against the expectations" err = ErrorBundle(None, True) package = XPIManager(open(file_, "rb"), file_) contents = package.get_file_data() # We need to have an install.rdf. assert "install.rdf" in contents # Load up the install.rdf into an RDFParser install_file = package.read("install.rdf") install_rdf = RDFParser(install_file) results = typedetection.detect_type(err, install_rdf, package) assert results == expectation if not failure: assert not err.failed() else: assert err.failed()
def test_skip_blacklisted_file(): """Ensure blacklisted files are skipped for processing.""" package_data = open('tests/resources/libraryblacklist/errors.xpi') package = XPIManager(package_data, mode='r', name='errors.xpi') err = ErrorBundle() test_content.test_packed_packages(err, package) print err.print_summary() assert err.notices assert not err.failed()
def test_outdated_sdkversion(): """ Tests that add-ons using a version other than the latest release are warned against, but module hashes are still recognized. """ xpi = XPIManager("tests/resources/jetpack/jetpack-1.8-outdated.xpi") err = _do_test(xpi, allow_old_sdk=False) assert err.failed() # Make sure we don't have any version mismatch warnings eq_(len(err.warnings), 1) eq_(err.warnings[0]["id"][2], "outdated_version")
def test_validate_libs_in_compat_mode(): xpi = 'tests/resources/libraryblacklist/addon_with_mootools.xpi' with open(xpi) as data: package = XPIManager(data, mode='r', name='addon_with_mootools.xpi') appversions = { FIREFOX_GUID: version_range('firefox', '39.0a1', '39.*') } err = ErrorBundle(for_appversions=appversions) test_content.test_packed_packages(err, package) assert err.get_resource('scripts'), ( 'expected mootools scripts to be marked for proessing') assert err.get_resource('scripts')[0]['scripts'] == set( ['content/mootools.js'])
def test_package(err, file_, name, expectation=PACKAGE_ANY): "Begins tests for the package." # Load up a new instance of an XPI. package = XPIManager(file_, name) if not package.zf: # Die on this one because the file won't open. return err.error(("main", "test_package", "unopenable"), "The XPI could not be opened.") # Test the XPI file for corruption. if package.test(): err.reject = True return err.error(("main", "test_package", "corrupt"), "XPI package appears to be corrupt.") if package.extension in assumed_extensions: assumed_type = assumed_extensions[package.extension] # Is the user expecting a different package type? if not expectation in (PACKAGE_ANY, assumed_type): err.error(("main", "test_package", "unexpected_type"), "Unexpected package type (found theme)") # Cache a copy of the package contents. package_contents = package.get_file_data() # Test the install.rdf file to see if we can get the type that way. has_install_rdf = "install.rdf" in package_contents if has_install_rdf: _load_install_rdf(err, package, expectation) return test_inner_package(err, package_contents, package)
def _test_type(file_, expectation, failure=False): 'Tests a file against the expectations' err = ErrorBundle(None, True) package = XPIManager(open(file_), mode='r', name=file_) contents = package.package_contents() # We need to have an install.rdf. assert 'install.rdf' in contents # Load up the install.rdf into an RDFParser install_file = package.read('install.rdf') install_rdf = RDFParser(err, install_file) results = typedetection.detect_type(err, install_rdf, package) assert results == expectation if not failure: assert not err.failed() else: assert err.failed() return err
def _do_test(path, test, failure=True, require_install=False, set_type=0, listed=False, xpi_mode='r'): package_data = open(path, 'rb') package = XPIManager(package_data, mode=xpi_mode, name=path) err = ErrorBundle() if listed: err.save_resource('listed', True) # Populate in the dependencies. if set_type: err.detected_type = set_type # Conduit test requires type if require_install: if 'install.rdf' in package: err.save_resource('has_install_rdf', True) rdf_data = package.read('install.rdf') install_rdf = RDFParser(err, rdf_data) err.save_resource('install_rdf', install_rdf) elif 'manifest.json' in package: err.save_resource('has_manifest_json', True) manifest_data = package.read('manifest.json') manifest_json = ManifestJsonParser(err, manifest_data) err.save_resource('install_rdf', manifest_json) populate_chrome_manifest(err, package) test(err, package) print err.print_summary(verbose=True) if failure: assert err.failed() else: assert not err.failed() return err
def test_package(err, file_, name, expectation=PACKAGE_ANY, for_appversions=None): 'Begins tests for the package.' # Load up a new instance of an XPI. try: package = XPIManager(file_, mode='r', name=name) has_package_json = 'package.json' in package has_manifest_json = 'manifest.json' in package has_install_rdf = 'install.rdf' in package # install.rdf? | package.json? | manifest.json? | error | use-file # Yes | No | No | No | install.rdf # Yes | Yes | No | No | install.rdf # Yes | No | Yes | No | install.rdf # No | No | Yes | No | manifest.json # No | No | No | Yes | install.rdf # No | Yes | No | No | package.json # No | No | Yes | Yes | install.rdf if has_package_json: _load_package_json(err, package, expectation) if has_manifest_json: _load_manifest_json(err, package, expectation) if has_install_rdf: _load_install_rdf(err, package, expectation) except IOError: # Die on this one because the file won't open. err.error(('main', 'test_package', 'unopenable'), 'The XPI could not be opened.') return except (BadZipfile, zlib_error): # Die if the zip file is corrupt. err.error(('submain', '_load_install_rdf', 'badzipfile'), error='Corrupt ZIP file', description='We were unable to decompress the zip file.') return if package.extension in assumed_extensions: assumed_type = assumed_extensions[package.extension] # Is the user expecting a different package type? if expectation not in (PACKAGE_ANY, assumed_type): err.error(('main', 'test_package', 'unexpected_type'), 'Unexpected package type (found theme)') test_inner_package(err, package, for_appversions)
def test_package(err, file_, name, expectation=PACKAGE_ANY, for_appversions=None): "Begins tests for the package." # Load up a new instance of an XPI. try: package = XPIManager(file_, mode="r", name=name) except: # Die on this one because the file won't open. return err.error(("main", "test_package", "unopenable"), "The XPI could not be opened.") # Test the XPI file for corruption. if package.test(): return err.error(("main", "test_package", "corrupt"), "XPI package appears to be corrupt.") if package.extension in assumed_extensions: assumed_type = assumed_extensions[package.extension] # Is the user expecting a different package type? if not expectation in (PACKAGE_ANY, assumed_type): err.error(("main", "test_package", "unexpected_type"), "Unexpected package type (found theme)") # Test the install.rdf file to see if we can get the type that way. has_install_rdf = "install.rdf" in package if has_install_rdf: _load_install_rdf(err, package, expectation) return test_inner_package(err, package, for_appversions)
def _run_test(filename, expectation, should_fail=True): name = "tests/resources/submain/%s" % filename pack = open(name) xpi = XPIManager(pack, mode="r", name=name) err = ErrorBundle(None, True) submain._load_install_rdf(err, xpi, expectation) if should_fail: assert err.failed() else: assert not err.failed() assert err.get_resource("install_rdf") return err
def test_comparer(): "Tests the function that compares two packages." ref = XPIManager("tests/resources/l10n/langpack/reference.jar") ref.locale_name = "en-US" extra_ref = XPIManager( "tests/resources/l10n/langpack/extra_files_ref.jar") pass_ = XPIManager("tests/resources/l10n/langpack/pass.jar") pass_.locale_name = "en-US" mfile = XPIManager( "tests/resources/l10n/langpack/missing_file.jar") mfile.locale_name = "en-US" extra = XPIManager( "tests/resources/l10n/langpack/extra_files.jar") extra.locale_name = "en-US" mfileent = XPIManager( "tests/resources/l10n/langpack/missing_file_entities.jar") mfileent.locale_name = "en-US" ment = XPIManager( "tests/resources/l10n/langpack/missing_entities.jar") ment.locale_name = "en-US" assert _compare_packs(ref, pass_) == 3 assert _compare_packs(extra_ref, pass_) == 3 assert _compare_packs(ref, extra) == 3 assert _compare_packs(ref, mfile) == 4 assert _compare_packs(ref, mfileent) == 3 assert _compare_packs(ref, ref) > 3
def test_read_file(): """Test that a file can be read from the package.""" z = XPIManager(get_path('xpi/install_rdf_only.xpi')) assert z.read('install.rdf') is not None
def test_valid_name(): "Test that the manager can retrieve the correct file name." z = XPIManager(get_path('xpi/install_rdf_only.xpi')) contents = z.package_contents() assert 'install.rdf' in contents
def test_lp_xpi(err, xpi_package): "Tests a language pack for L10n completeness" # Don't even both with the test(s) if there's no chrome.manifest. if "chrome.manifest" not in xpi_package: return None locales = _get_locales(err) # Get the reference packages. references = [] support_references = err.get_resource("supports") if not support_references: references.append("firefox") err.info(("testcases_l10ncompleteness", "test_lp_xpi", "missing_app_support"), "Supported app missing in localization completeness.", "While testing in localization comleteness, a list of " "supported applications for the language pack was not found. " "This is likely because there are no listed " "<em:targetApplication> elements in the install.rdf file.") else: for support in support_references: ref_xpi = XPIManager( os.path.join(os.path.dirname(__file__), "langpacks/%s.xpi" % support)) ref_xpi.app_name = support reference_locales = _get_locales(None, ref_xpi) references.append((ref_xpi, reference_locales)) # Iterate each supported reference package for (ref_xpi, ref_locales) in references: # Iterate each locale in each supported reference package ref_pack = _get_locale_manager(err, ref_xpi, { "path": "en-US.jar", "jarred": True }, no_cache=True) for ref_locale_name in ref_locales: ref_locale = ref_locales[ref_locale_name] ref_predicate = ref_locale["predicate"] corresp_locales = [ locales[name] for name in locales if locales[name]["predicate"] == ref_predicate ] # If we found no matching locale, then it's missing from the pack if not corresp_locales: err.warning( ("testcases_l10ncompleteness", "test_lp_xpi", "find_corresponding_locale"), "Could not find corresponding locale", [ "A locale was found in the reference package, " "however it was not found in the target package.", "Missing locale: %s" % ref_predicate ], filename="chrome.manifest") continue target_locale = corresp_locales[0] target_pack = _get_locale_manager(err, xpi_package, target_locale) if target_pack is None: continue results = _compare_packages(reference=ref_pack, target=target_pack, ref_base=ref_locale["target"], locale_base=target_locale["target"]) # Report the findings after each supported app's locale _aggregate_results(err, results, target_locale) # Clear the cache at the end of the test LOCALE_CACHE = {}
def test_lp_xpi(err, xpi_package): "Tests a language pack for L10n completeness" # Don't even both with the test(s) if there's no chrome.manifest. if "chrome.manifest" not in xpi_package: return None locales = _get_locales(err) # Get the reference packages. references = [] support_references = err.get_resource("supports") if not support_references: references.append("firefox") err.info( ("testcases_l10ncompleteness", "test_lp_xpi", "missing_app_support"), "Supported app missing in localization completeness.", "While testing in localization comleteness, a list of " "supported applications for the language pack was not found. " "This is likely because there are no listed " "<em:targetApplication> elements in the install.rdf file.", ) else: for support in support_references: ref_xpi = XPIManager(os.path.join(os.path.dirname(__file__), "langpacks/%s.xpi" % support)) ref_xpi.app_name = support reference_locales = _get_locales(None, ref_xpi) references.append((ref_xpi, reference_locales)) # Iterate each supported reference package for (ref_xpi, ref_locales) in references: # Iterate each locale in each supported reference package ref_pack = _get_locale_manager(err, ref_xpi, {"path": "en-US.jar", "jarred": True}, no_cache=True) for ref_locale_name in ref_locales: ref_locale = ref_locales[ref_locale_name] ref_predicate = ref_locale["predicate"] corresp_locales = [locales[name] for name in locales if locales[name]["predicate"] == ref_predicate] # If we found no matching locale, then it's missing from the pack if not corresp_locales: err.warning( ("testcases_l10ncompleteness", "test_lp_xpi", "find_corresponding_locale"), "Could not find corresponding locale", [ "A locale was found in the reference package, " "however it was not found in the target package.", "Missing locale: %s" % ref_predicate, ], filename="chrome.manifest", ) continue target_locale = corresp_locales[0] target_pack = _get_locale_manager(err, xpi_package, target_locale) if target_pack is None: continue results = _compare_packages( reference=ref_pack, target=target_pack, ref_base=ref_locale["target"], locale_base=target_locale["target"], ) # Report the findings after each supported app's locale _aggregate_results(err, results, target_locale) # Clear the cache at the end of the test LOCALE_CACHE = {}
def test_get_list(): "Test that the manager can read the file listing" z = XPIManager("tests/resources/xpi/install_rdf_only.xpi") assert z.get_file_data()
def _process_file(err, xpi_package, name, file_data, name_lower, pollutable=False): """Process a single file's content tests.""" extension = os.path.splitext(name_lower)[1] # If that item is a container file, unzip it and scan it. if extension == '.jar': # This is either a subpackage or a nested theme. is_subpackage = not err.get_resource('is_multipackage') # Unpack the package and load it up. package = StringIO(file_data) try: sub_xpi = XPIManager(package, mode='r', name=name, subpackage=is_subpackage) except BadZipfile: err.error(('testcases_content', 'test_packed_packages', 'jar_subpackage_corrupt'), 'Subpackage corrupt.', 'The subpackage appears to be corrupt, and could not ' 'be opened.', name) return # Let the error bunder know we're in a sub-package. err.push_state(name) err.detected_type = (PACKAGE_SUBPACKAGE if is_subpackage else PACKAGE_THEME) err.set_tier(1) supported_versions = (err.supported_versions.copy() if err.supported_versions else err.supported_versions) if is_subpackage: testendpoint_validator.test_inner_package(err, sub_xpi) else: testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) err.supported_versions = supported_versions elif extension == '.xpi': # It's not a subpackage, it's a nested extension. These are # found in multi-extension packages. # Unpack! package = StringIO(file_data) err.push_state(name_lower) err.set_tier(1) # There are no expected types for packages within a multi- # item package. testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) # Reset to the current tier else: if not file_data: return # Convert the file data to unicode. file_data = unicodehelper.decode(file_data) if extension in ('.js', '.jsm'): testendpoint_js.test_js_file(err, name, file_data, pollutable=pollutable) elif extension == '.css': testendpoint_css.test_css_file(err, name, file_data) run_regex_tests(file_data, err, filename=name)
def test_package(err, file_, name, expectation=PACKAGE_ANY, for_appversions=None): 'Begins tests for the package.' # Load up a new instance of an XPI. try: package = XPIManager(file_, mode='r', name=name) has_package_json = 'package.json' in package has_manifest_json = 'manifest.json' in package has_install_rdf = 'install.rdf' in package # install.rdf? | package.json? | manifest.json? | error | use-file # Yes | No | No | No | install.rdf # Yes | Yes | No | No | install.rdf # Yes | No | Yes | No | install.rdf # No | No | Yes | No | manifest.json # No | No | No | Yes | install.rdf # No | Yes | No | No | package.json # No | No | Yes | Yes | install.rdf if has_package_json: _load_package_json(err, package, expectation) if has_manifest_json: _load_manifest_json(err, package, expectation) if has_install_rdf: _load_install_rdf(err, package, expectation) except IOError: # Die on this one because the file won't open. err.error(('main', 'test_package', 'unopenable'), 'The XPI could not be opened.') return except (BadZipfile, zlib_error): # Die if the zip file is corrupt. err.error(('submain', '_load_install_rdf', 'badzipfile'), error='Corrupt ZIP file', description='We were unable to decompress the zip file.') return if package.extension in assumed_extensions: assumed_type = assumed_extensions[package.extension] # Is the user expecting a different package type? if expectation not in (PACKAGE_ANY, assumed_type): err.error(('main', 'test_package', 'unexpected_type'), 'Unexpected package type (found theme)') if (err.get_resource('has_manifest_json') and not err.get_resource('has_package_json') and not err.get_resource('has_install_rdf')): # It's a WebExtension. Those are not supported by amo-validator, we # should be using the linter instead. Only reason we could be using # amo validator with a WebExtension is if a developer tried to use # the compatibilty checker with a WebExtension, so add a relevant error # message. format_args = {'link': MDN_DOC % 'Mozilla/Add-ons/WebExtensions'} err.error( ('submain', 'test_inner_package', 'webextension'), 'This tool only works with legacy add-ons. See {link} for more ' 'information about WebExtension APIs.'.format(**format_args)) else: test_inner_package(err, package, for_appversions)
def packager(data, xpi_path, features): """Package an add-on from input data. The resulting package will be saved as xpi_path. data format: - id : <em:id> value - version : <em:version> - name : <em:name> - description : <em:description> - author_name : <em:author> - contributors : \n-delimited list of contributors - targetapplications : Dict in the form of: { "min_ver": "3.6", "max_ver": "6.0a1", "guid": "...", } - uuid : A UUID value that is unique to this package - slug : A slug value based on the name which will be used as a package identifier. xpi_path should be the file path to build the XPI at. features should be a set containing string names of each of the features to include. """ # Sanitize the slug. data['slug'] = _slugify(data.get('slug', '')) # Instantiate the XPI Manager. from validator.xpi import XPIManager xpi = XPIManager(xpi_path, mode='w') xpi.write('install.rdf', build_installrdf(data, features)) xpi.write('chrome.manifest', build_chrome_manifest(data, features)) # Sanitize all the input after building `install.rdf` (which is escaped # when jinja renders the template) to prevent doubly escaping the input. data = escape_all(data) _write_resource('defaults/preferences/prefs.js', xpi, data) _write_resource('chrome/skin/overlay.css', xpi, data) _write_resource('chrome/locale/en-US/overlay.dtd', xpi, data) _write_resource('chrome/locale/en-US/overlay.properties', xpi, data) if 'about_dialog' in features: _write_resource('chrome/content/about.xul', xpi, data) _write_resource('chrome/locale/en-US/about.dtd', xpi) if 'preferences_dialog' in features: _write_resource('chrome/content/options.xul', xpi, data) _write_resource('chrome/locale/en-US/options.dtd', xpi, data) if 'toolbar_button' in features: _write_resource('chrome/skin/toolbar-button.png', xpi) if 'sidebar_support' in features: _write_resource('chrome/content/ff-sidebar.js', xpi) _write_resource('chrome/content/ff-sidebar.xul', xpi, data) xpi.write('chrome/content/ff-overlay.xul', build_ffoverlay_xul(data, features)) _write_resource('chrome/content/ff-overlay.js', xpi, data) xpi.zf.close() return xpi_path
def test_comparer(): 'Tests the function that compares two packages.' ref = XPIManager('tests/resources/l10n/langpack/reference.jar') ref.locale_name = 'en-US' extra_ref = XPIManager('tests/resources/l10n/langpack/extra_files_ref.jar') pass_ = XPIManager('tests/resources/l10n/langpack/pass.jar') pass_.locale_name = 'en-US' mfile = XPIManager('tests/resources/l10n/langpack/missing_file.jar') mfile.locale_name = 'en-US' extra = XPIManager('tests/resources/l10n/langpack/extra_files.jar') extra.locale_name = 'en-US' mfileent = XPIManager( 'tests/resources/l10n/langpack/missing_file_entities.jar') mfileent.locale_name = 'en-US' ment = XPIManager('tests/resources/l10n/langpack/missing_entities.jar') ment.locale_name = 'en-US' assert _compare_packs(ref, pass_) == 3 assert _compare_packs(extra_ref, pass_) == 3 assert _compare_packs(ref, extra) == 3 assert _compare_packs(ref, mfile) == 4 assert _compare_packs(ref, mfileent) == 3 assert _compare_packs(ref, ref) > 3
def test_read_file(): "Test that a file can be read from the package" z = XPIManager("tests/resources/xpi/install_rdf_only.xpi") assert z.read("install.rdf") is not None
def test_valid_name(): "Test that the manager can retrieve the correct file name" z = XPIManager("tests/resources/xpi/install_rdf_only.xpi") contents = z.get_file_data() assert "install.rdf" in contents assert z.test() == False
def test_get_list(): "Test that the manager can read the file listing" z = XPIManager("tests/resources/xpi/install_rdf_only.xpi") assert z.package_contents()
def __init__(self, filename, project=None, release=None, name=None): """ Fills in a list of locales from the chrome.manifest file. """ Bundle.__init__(self, project, release) self.xpi = XPIManager(filename, name=name) # here we will store managers for jarfiles self.jarfiles = {} chrome = ChromeManifest(self.xpi.read("chrome.manifest"), "manifest") locales = list(chrome.get_triples("locale")) if not locales: return None # read the list for locale in locales: code, location = locale["object"].split() # finding out the language of the locale try: lang = self._get_lang(code) except Language.DoesNotExist: self.log("Locale %s SKIPPED" % code, "font-weight:bold") continue # Locales can be bundled in JARs jarred = location.startswith("jar:") if jarred: # We just care about the JAR path location = location[4:] split_location = location.split("!", 2) # Ignore malformed JAR URIs. if len(split_location) < 2: continue jarname, location = split_location # missing file mentioned if jarname not in self.xpi: continue # may be we have already read this one if jarname in self.jarfiles: package = self.jarfiles[jarname] else: jar = StringIO(self.xpi.read(jarname)) package = XPIManager(jar, mode="r", name=jarname) else: package = self.xpi # and now we read files from there location = location.strip('/') result = {} for f in package.package_contents(): f = f.strip("/") if f.startswith(location) and f != location: result[f.split("/")[-1]] = package.read(f) # file with same name in different jars can get overwritten if lang not in self.locales: self.locales[lang] = result else: self.locales[lang].update(result)
def test_open(): """Test that the manager will open the package.""" z = XPIManager(get_path('xpi/install_rdf_only.xpi')) assert z is not None
def test_valid_name(): 'Test that the manager can retrieve the correct file name.' z = XPIManager(get_path('xpi/install_rdf_only.xpi')) contents = z.package_contents() assert 'install.rdf' in contents
def test_valid_name(): "Test that the manager can retrieve the correct file name." z = XPIManager(get_path("xpi/install_rdf_only.xpi")) contents = z.package_contents() assert "install.rdf" in contents assert z.test() == False
def test_packed_packages(err, package_contents=None, xpi_package=None): "Tests XPI and JAR files for naughty content." processed_files = 0 hash_whitelist = [x[:-1] for x in open(os.path.join(os.path.dirname(__file__), 'whitelist_hashes.txt')).readlines()] # Iterate each item in the package. for name, data in package_contents.items(): if name.startswith("__MACOSX") or \ name.startswith(".DS_Store"): continue if name.split("/")[-1].startswith("._"): err.notice(("testcases_content", "test_packed_packages", "macintosh_junk"), "Garbage file found.", ["""A junk file has been detected. It may cause problems with proper operation of the add-on down the road.""", "It is recommended that you delete the file"], name) try: file_data = xpi_package.read(name) except KeyError: # pragma: no cover _read_error(err, name) # Skip over whitelisted hashes hash = hashlib.sha1(file_data).hexdigest() if hash in hash_whitelist: continue processed = False # If that item is a container file, unzip it and scan it. if data["extension"] == "jar": # This is either a subpackage or a nested theme. # Whether this is a subpackage or a nested theme is # determined by whether it is in the root folder or not. # Subpackages are always found in a directory such as # /chrome or /content. is_subpackage = name.count("/") > 0 # Unpack the package and load it up. package = StringIO(file_data) sub_xpi = XPIManager(package, name, is_subpackage) if not sub_xpi.zf: err.error(("testcases_content", "test_packed_packages", "jar_subpackage_corrupt"), "Subpackage corrupt.", """The subpackage could not be opened due to issues with corruption. Ensure that the file is valid.""", name) continue temp_contents = sub_xpi.get_file_data() # Let the error bunder know we're in a sub-package. err.push_state(data["name_lower"]) err.set_type(PACKAGE_SUBPACKAGE) # Subpackage testendpoint_validator.test_inner_package(err, temp_contents, sub_xpi) err.tier = 2 package.close() err.pop_state() elif data["extension"] == "xpi": # It's not a subpackage, it's a nested extension. These are # found in multi-extension packages. # Unpack! package = StringIO(file_data) err.push_state(data["name_lower"]) # There are no expected types for packages within a multi- # item package. testendpoint_validator.test_package(err, package, name) err.tier = 2 # Reset to the current tier package.close() err.pop_state() elif data["extension"] in ("xul", "xml", "html", "xhtml"): parser = testendpoint_markup.MarkupParser(err) parser.process(name, charsethelper.decode(file_data), data["extension"]) processed = True elif data["extension"] in ("css", "js", "jsm"): if not file_data: continue file_data = charsethelper.decode(file_data) if data["extension"] == "css": testendpoint_css.test_css_file(err, name, file_data) elif data["extension"] in ("js", "jsm"): testendpoint_js.test_js_file(err, name, file_data) # This is tested in test_langpack.py if err.detected_type == PACKAGE_LANGPACK and not processed: testendpoint_langpack.test_unsafe_html(err, name, file_data) # This aids in creating unit tests. processed_files += 1 return processed_files
def test_lp_xpi(err, xpi_package): 'Tests a language pack for L10n completeness' # Don't even both with the test(s) if there's no chrome.manifest. if 'chrome.manifest' not in xpi_package: return None locales = _get_locales(err) # Get the reference packages. references = [] support_references = err.get_resource('supports') if not support_references: references.append('firefox') err.info(('testcases_l10ncompleteness', 'test_lp_xpi', 'missing_app_support'), 'Supported app missing in localization completeness.', 'While testing in localization comleteness, a list of ' 'supported applications for the language pack was not found. ' 'This is likely because there are no listed ' '<em:targetApplication> elements in the install.rdf file.') else: for support in support_references: ref_xpi = XPIManager(os.path.join(os.path.dirname(__file__), 'langpacks/%s.xpi' % support)) ref_xpi.app_name = support reference_locales = _get_locales(None, ref_xpi) references.append((ref_xpi, reference_locales)) # Iterate each supported reference package for (ref_xpi, ref_locales) in references: # Iterate each locale in each supported reference package ref_pack = _get_locale_manager(err, ref_xpi, {'path': 'en-US.jar', 'jarred': True}, no_cache=True) for ref_locale_name in ref_locales: ref_locale = ref_locales[ref_locale_name] ref_predicate = ref_locale['predicate'] corresp_locales = [locales[name] for name in locales if locales[name]['predicate'] == ref_predicate] # If we found no matching locale, then it's missing from the pack if not corresp_locales: err.warning(('testcases_l10ncompleteness', 'test_lp_xpi', 'find_corresponding_locale'), 'Could not find corresponding locale', ['A locale was found in the reference package, ' 'however it was not found in the target package.', 'Missing locale: %s' % ref_predicate], filename='chrome.manifest') continue target_locale = corresp_locales[0] target_pack = _get_locale_manager(err, xpi_package, target_locale) if target_pack is None: continue results = _compare_packages(reference=ref_pack, target=target_pack, ref_base=ref_locale['target'], locale_base=target_locale['target']) # Report the findings after each supported app's locale _aggregate_results(err, results, target_locale) # Clear the cache at the end of the test LOCALE_CACHE.clear()
def _process_file(err, xpi_package, name, file_data, name_lower, pollutable=False): """Process a single file's content tests.""" # If that item is a container file, unzip it and scan it. if name_lower.endswith(".jar"): # This is either a subpackage or a nested theme. is_subpackage = not err.get_resource("is_multipackage") # Unpack the package and load it up. package = StringIO(file_data) try: sub_xpi = XPIManager(package, mode="r", name=name, subpackage=is_subpackage) except Exception: err.error(("testcases_content", "test_packed_packages", "jar_subpackage_corrupt"), "Subpackage corrupt.", "The subpackage could not be opened due to issues " "with corruption. Ensure that the file is valid.", name) return None # Let the error bunder know we're in a sub-package. err.push_state(name) err.detected_type = (PACKAGE_SUBPACKAGE if is_subpackage else PACKAGE_THEME) err.set_tier(1) supported_versions = (err.supported_versions.copy() if err.supported_versions else err.supported_versions) if is_subpackage: testendpoint_validator.test_inner_package(err, sub_xpi) else: testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) err.supported_versions = supported_versions elif name_lower.endswith(".xpi"): # It's not a subpackage, it's a nested extension. These are # found in multi-extension packages. # Unpack! package = StringIO(file_data) err.push_state(name_lower) err.set_tier(1) # There are no expected types for packages within a multi- # item package. testendpoint_validator.test_package(err, package, name) err.pop_state() err.set_tier(2) # Reset to the current tier elif name_lower.endswith((".css", ".js", ".jsm")): if not file_data: return None # Convert the file data to unicode file_data = unicodehelper.decode(file_data) is_js = False if name_lower.endswith(".css"): testendpoint_css.test_css_file(err, name, file_data) elif name_lower.endswith((".js", ".jsm")): is_js = True testendpoint_js.test_js_file(err, name, file_data, pollutable=pollutable) run_regex_tests(file_data, err, name, is_js=is_js) return True return False
def test_lp_xpi(err, xpi_package): 'Tests a language pack for L10n completeness' # Don't even both with the test(s) if there's no chrome.manifest. if 'chrome.manifest' not in xpi_package: return None locales = _get_locales(err) # Get the reference packages. references = [] support_references = err.get_resource('supports') if not support_references: references.append('firefox') err.info(('testcases_l10ncompleteness', 'test_lp_xpi', 'missing_app_support'), 'Supported app missing in localization completeness.', 'While testing in localization comleteness, a list of ' 'supported applications for the language pack was not found. ' 'This is likely because there are no listed ' '<em:targetApplication> elements in the install.rdf file.') else: for support in support_references: ref_xpi = XPIManager( os.path.join(os.path.dirname(__file__), 'langpacks/%s.xpi' % support)) ref_xpi.app_name = support reference_locales = _get_locales(None, ref_xpi) references.append((ref_xpi, reference_locales)) # Iterate each supported reference package for (ref_xpi, ref_locales) in references: # Iterate each locale in each supported reference package ref_pack = _get_locale_manager(err, ref_xpi, { 'path': 'en-US.jar', 'jarred': True }, no_cache=True) for ref_locale_name in ref_locales: ref_locale = ref_locales[ref_locale_name] ref_predicate = ref_locale['predicate'] corresp_locales = [ locales[name] for name in locales if locales[name]['predicate'] == ref_predicate ] # If we found no matching locale, then it's missing from the pack if not corresp_locales: err.warning( ('testcases_l10ncompleteness', 'test_lp_xpi', 'find_corresponding_locale'), 'Could not find corresponding locale', [ 'A locale was found in the reference package, ' 'however it was not found in the target package.', 'Missing locale: %s' % ref_predicate ], filename='chrome.manifest') continue target_locale = corresp_locales[0] target_pack = _get_locale_manager(err, xpi_package, target_locale) if target_pack is None: continue results = _compare_packages(reference=ref_pack, target=target_pack, ref_base=ref_locale['target'], locale_base=target_locale['target']) # Report the findings after each supported app's locale _aggregate_results(err, results, target_locale) # Clear the cache at the end of the test LOCALE_CACHE.clear()