def _do_test(path, test, failure=True,
require_install=False, set_type=0):
package_data = open(path, "rb")
package = XPIManager(package_data, path)
contents = package.get_file_data()
err = ErrorBundle()
# Populate in the dependencies.
if set_type:
err.set_type(set_type) # Conduit test requires type
if require_install:
err.save_resource("has_install_rdf", True)
rdf_data = package.read("install.rdf")
install_rdf = RDFParser(rdf_data)
err.save_resource("install_rdf", install_rdf)
test(err, contents, package)
print err.print_summary(verbose=True)
if failure:
assert err.failed()
else:
assert not err.failed()
return err
def _do_test(path,
test,
failure=True,
require_install=False,
set_type=0,
listed=False,
xpi_mode='r'):
package_data = open(path, 'rb')
package = XPIManager(package_data, mode=xpi_mode, name=path)
err = ErrorBundle()
if listed:
err.save_resource('listed', True)
# Populate in the dependencies.
if set_type:
err.detected_type = set_type # Conduit test requires type
if require_install:
err.save_resource('has_install_rdf', True)
rdf_data = package.read('install.rdf')
install_rdf = RDFParser(err, rdf_data)
err.save_resource('install_rdf', install_rdf)
populate_chrome_manifest(err, package)
test(err, package)
print err.print_summary(verbose=True)
if failure:
assert err.failed()
else:
assert not err.failed()
return err
def _do_test(path, test, failure=True,
require_install=False, set_type=0,
listed=False, xpi_mode="r"):
package_data = open(path, "rb")
package = XPIManager(package_data, mode=xpi_mode, name=path)
err = ErrorBundle()
if listed:
err.save_resource("listed", True)
# Populate in the dependencies.
if set_type:
err.detected_type = set_type # Conduit test requires type
if require_install:
err.save_resource("has_install_rdf", True)
rdf_data = package.read("install.rdf")
install_rdf = RDFParser(err, rdf_data)
err.save_resource("install_rdf", install_rdf)
populate_chrome_manifest(err, package)
test(err, package)
print err.print_summary(verbose=True)
if failure:
assert err.failed()
else:
assert not err.failed()
return err
def test_bad_file():
"Tests that the XPI manager correctly reports a bad XPI file."
x = XPIManager("tests/resources/junk.xpi")
assert not x.zf
x = XPIManager("tests/resources/corrupt.xpi")
assert x.test()
def test_get_list():
"""Test that the manager can read the file listing."""
z = XPIManager(get_path('xpi/install_rdf_only.xpi'))
assert not z.contents_cache
assert z.package_contents()
assert z.contents_cache # Spelling check!
z.contents_cache = 'foo'
eq_(z.package_contents(), 'foo')
def test_get_list():
"""Test that the manager can read the file listing."""
z = XPIManager(get_path('xpi/install_rdf_only.xpi'))
assert not z.contents_cache
assert z.package_contents()
assert z.contents_cache # Spelling check!
z.contents_cache = 'foo'
eq_(z.package_contents(), 'foo')
def test_bad_file():
"""Tests that the XPI manager correctly reports a bad XPI file."""
try:
x = XPIManager(get_path("junk.xpi"))
except BadZipfile:
pass
x = XPIManager(get_path("corrupt.xpi"))
assert x.test()
def test_get_list():
"Test that the manager can read the file listing"
z = XPIManager("tests/resources/xpi/install_rdf_only.xpi")
assert not z.contents_cache
assert z.package_contents()
assert z.contents_cache # Spelling check!
z.contents_cache = "foo"
eq_(z.package_contents(), "foo")
def test_bad_file():
"Tests that the XPI manager correctly reports a bad XPI file."
try:
x = XPIManager("tests/resources/junk.xpi")
except zipfile.BadZipfile:
pass
x = XPIManager("tests/resources/corrupt.xpi")
assert x.test()
def test_write_file():
"""Test that a file can be written in UTF-8 to the package."""
with tempfile.NamedTemporaryFile(delete=False) as t:
temp_fn = t.name
try:
z = XPIManager(temp_fn, mode='w')
f, d = 'install.rdf', '注目のコレクション'.decode('utf-8')
z.write(f, d)
eq_(z.read(f), d.encode('utf-8'))
finally:
os.unlink(temp_fn)
def test_write_file():
"""Test that a file can be written in UTF-8 to the package."""
with tempfile.NamedTemporaryFile(delete=False) as t:
temp_fn = t.name
try:
z = XPIManager(temp_fn, mode='w')
f, d = 'install.rdf', '注目のコレクション'.decode('utf-8')
z.write(f, d)
eq_(z.read(f), d.encode('utf-8'))
finally:
os.unlink(temp_fn)
def clean_xpifile(self):
"Make sure that the uploaded file is a valid XPI file."
xpifile = self.cleaned_data['xpifile']
if xpifile:
try:
xpi = XPIManager(xpifile, name=xpifile.name)
if xpi.test():
raise
except:
raise forms.ValidationError(_("File doesn't seem to be valid XPI"))
return xpifile
def test_blacklisted_files():
"""Tests the validator's ability to hash each individual file and
(based on this information) determine whether the addon passes or
fails the validation process."""
package_data = open("tests/resources/libraryblacklist/blocked.xpi")
package = XPIManager(package_data, "blocked.xpi")
contents = package.get_file_data()
err = ErrorBundle()
libblacklist.test_library_blacklist(err, contents, package)
print err.print_summary()
assert err.notices
def test_package(err, file_, name, expectation=PACKAGE_ANY,
for_appversions=None):
"Begins tests for the package."
# Load up a new instance of an XPI.
try:
package = XPIManager(file_, mode="r", name=name)
except:
# Die on this one because the file won't open.
return err.error(("main",
"test_package",
"unopenable"),
"The XPI could not be opened.")
# Test the XPI file for corruption.
if package.test():
return err.error(("main",
"test_package",
"corrupt"),
"XPI package appears to be corrupt.")
if package.extension in assumed_extensions:
assumed_type = assumed_extensions[package.extension]
# Is the user expecting a different package type?
if not expectation in (PACKAGE_ANY, assumed_type):
err.error(("main",
"test_package",
"unexpected_type"),
"Unexpected package type (found theme)")
# Test the install.rdf file to see if we can get the type that way.
has_install_rdf = "install.rdf" in package
if has_install_rdf:
_load_install_rdf(err, package, expectation)
try:
output = test_inner_package(err, package, for_appversions)
except ValidationTimeout as ex:
err.error(
err_id=("main", "test_package", "timeout"),
error="Validation timed out",
description=["The validation process took too long to "
"complete. Contact an addons.mozilla.org editor "
"for more information.",
str(ex)])
output = None
return output
def _get_locale_manager(err, xpi_package, description, no_cache=False):
'Returns the XPIManager object for a locale'
if not description['jarred']:
return xpi_package
path = description['path']
if path in LOCALE_CACHE and not no_cache:
return LOCALE_CACHE[path]
if path not in xpi_package:
# TODO: Pass the filename of the triple's manifest.
err.warning(
('testcases_l10ncompleteness', '_get_locale_manager',
'manager_absent'),
'Listed locale does not exist', [
'A locale JAR is listed in chrome.manifest, but it could '
'not be located. Check the spelling and capitalization '
'of the path.',
'Missing JAR: %s' % path
],
filename='chrome.manifest')
return None
jar = StringIO(xpi_package.read(path))
locale = XPIManager(jar, mode='r', name=path)
if not no_cache:
LOCALE_CACHE[path] = locale
return locale
def _get_locale_manager(err, xpi_package, description, no_cache=False):
"Returns the XPIManager object for a locale"
if not description["jarred"]:
return xpi_package
path = description["path"]
if path in LOCALE_CACHE and not no_cache:
return LOCALE_CACHE[path]
if path not in xpi_package:
# TODO: Pass the filename of the triple's manifest.
err.warning(
("testcases_l10ncompleteness", "_get_locale_manager",
"manager_absent"),
"Listed locale does not exist", [
"A locale JAR is listed in chrome.manifest, but it could "
"not be located. Check the spelling and capitalization "
"of the path.",
"Missing JAR: %s" % path
],
filename="chrome.manifest")
return None
jar = StringIO(xpi_package.read(path))
locale = XPIManager(jar, mode="r", name=path)
if not no_cache:
LOCALE_CACHE[path] = locale
return locale
def test_blacklisted_files():
"""
Tests the validator's ability to hash each individual file and (based on
this information) determine whether the addon passes or fails the
validation process.
"""
package_data = open('tests/resources/libraryblacklist/blocked.xpi')
package = XPIManager(package_data, mode='r', name='blocked.xpi')
err = ErrorBundle()
test_content.test_packed_packages(err, package)
print err.print_summary()
assert err.notices
assert not err.failed()
eq_(
err.metadata.get('identified_files'), {
'test.js': {
'path':
'This file is a false script to facilitate '
'testing of library blacklisting.'
}
})
def test_missing_file():
"""Tests that the XPI manager correctly reports a missing XPI file."""
passed = False
try:
x = XPIManager('foo.bar')
except:
passed = True
assert passed
def test_package(err,
file_,
name,
expectation=PACKAGE_ANY,
for_appversions=None):
"Begins tests for the package."
# Load up a new instance of an XPI.
try:
package = XPIManager(file_, mode="r", name=name)
has_package_json = "package.json" in package
has_install_rdf = "install.rdf" in package
# install.rdf? | package.json? | error | use-file
# Yes | No | No | install.rdf
# Yes | Yes | No | install.rdf
# No | No | Yes | install.rdf
# No | Yes | No | package.json
if has_package_json:
_load_package_json(err, package, expectation)
if has_install_rdf:
_load_install_rdf(err, package, expectation)
except IOError:
# Die on this one because the file won't open.
return err.error(("main", "test_package", "unopenable"),
"The XPI could not be opened.")
except (BadZipfile, zlib_error):
# Die if the zip file is corrupt.
return err.error(
("submain", "_load_install_rdf", "badzipfile"),
error="Corrupt ZIP file",
description="We were unable to decompress the zip file.")
if package.extension in assumed_extensions:
assumed_type = assumed_extensions[package.extension]
# Is the user expecting a different package type?
if not expectation in (PACKAGE_ANY, assumed_type):
err.error(("main", "test_package", "unexpected_type"),
"Unexpected package type (found theme)")
try:
output = test_inner_package(err, package, for_appversions)
except ValidationTimeout as ex:
err.error(err_id=("main", "test_package", "timeout"),
error="Validation timed out",
description=[
"The validation process took too long to "
"complete. Contact an addons.mozilla.org editor "
"for more information.",
str(ex)
])
output = None
return output
def test_mismatched_module_version():
"""
Tests that add-ons using modules from a version of the SDK
other than the version they claim.
"""
xpi = XPIManager('tests/resources/jetpack/jetpack-1.8-pretending-1.8.1.xpi')
err = _do_test(xpi)
assert err.failed()
assert any(w['id'][2] == 'mismatched_version' for w in err.warnings)
def test_new_module_location_spec():
"""
Tests that we don't fail for missing modules in add-ons generated with
newer versions of the SDK.
"""
xpi = XPIManager("tests/resources/jetpack/jetpack-1.14.xpi")
err = _do_test(xpi)
assert not any(w["id"][2] == "missing_jetpack_module"
for w in err.warnings)
def test_validate_libs_in_compat_mode():
xpi = "tests/resources/libraryblacklist/addon_with_mootools.xpi"
with open(xpi) as data:
package = XPIManager(data, mode="r", name="addon_with_mootools.xpi")
err = ErrorBundle(for_appversions=FX9_DEFINITION)
test_content.test_packed_packages(err, package)
assert err.get_resource("scripts"), (
"expected mootools scripts to be marked for proessing")
eq_(
err.get_resource("scripts")[0]["scripts"],
set(["content/mootools.js"]))
def test_future_sdkversion():
"""
Test that if the developer is using a verison of the SDK that's newer than
the latest recognized version, we don't throw an error.
"""
xpi = XPIManager("tests/resources/jetpack/jetpack-1.8-future.xpi")
err = _do_test(xpi, allow_old_sdk=False)
print err.print_summary(verbose=True)
assert not err.failed()
def _test_type(file_, expectation, failure=False):
"Tests a file against the expectations"
err = ErrorBundle(None, True)
package = XPIManager(open(file_, "rb"), file_)
contents = package.get_file_data()
# We need to have an install.rdf.
assert "install.rdf" in contents
# Load up the install.rdf into an RDFParser
install_file = package.read("install.rdf")
install_rdf = RDFParser(install_file)
results = typedetection.detect_type(err, install_rdf, package)
assert results == expectation
if not failure:
assert not err.failed()
else:
assert err.failed()
def test_skip_blacklisted_file():
"""Ensure blacklisted files are skipped for processing."""
package_data = open('tests/resources/libraryblacklist/errors.xpi')
package = XPIManager(package_data, mode='r', name='errors.xpi')
err = ErrorBundle()
test_content.test_packed_packages(err, package)
print err.print_summary()
assert err.notices
assert not err.failed()
def test_outdated_sdkversion():
"""
Tests that add-ons using a version other than the latest release
are warned against, but module hashes are still recognized.
"""
xpi = XPIManager("tests/resources/jetpack/jetpack-1.8-outdated.xpi")
err = _do_test(xpi, allow_old_sdk=False)
assert err.failed()
# Make sure we don't have any version mismatch warnings
eq_(len(err.warnings), 1)
eq_(err.warnings[0]["id"][2], "outdated_version")
def test_validate_libs_in_compat_mode():
xpi = 'tests/resources/libraryblacklist/addon_with_mootools.xpi'
with open(xpi) as data:
package = XPIManager(data, mode='r', name='addon_with_mootools.xpi')
appversions = {
FIREFOX_GUID: version_range('firefox', '39.0a1', '39.*')
}
err = ErrorBundle(for_appversions=appversions)
test_content.test_packed_packages(err, package)
assert err.get_resource('scripts'), (
'expected mootools scripts to be marked for proessing')
assert err.get_resource('scripts')[0]['scripts'] == set(
['content/mootools.js'])
def test_package(err, file_, name, expectation=PACKAGE_ANY):
"Begins tests for the package."
# Load up a new instance of an XPI.
package = XPIManager(file_, name)
if not package.zf:
# Die on this one because the file won't open.
return err.error(("main",
"test_package",
"unopenable"),
"The XPI could not be opened.")
# Test the XPI file for corruption.
if package.test():
err.reject = True
return err.error(("main",
"test_package",
"corrupt"),
"XPI package appears to be corrupt.")
if package.extension in assumed_extensions:
assumed_type = assumed_extensions[package.extension]
# Is the user expecting a different package type?
if not expectation in (PACKAGE_ANY, assumed_type):
err.error(("main",
"test_package",
"unexpected_type"),
"Unexpected package type (found theme)")
# Cache a copy of the package contents.
package_contents = package.get_file_data()
# Test the install.rdf file to see if we can get the type that way.
has_install_rdf = "install.rdf" in package_contents
if has_install_rdf:
_load_install_rdf(err, package, expectation)
return test_inner_package(err, package_contents, package)
def _test_type(file_, expectation, failure=False):
'Tests a file against the expectations'
err = ErrorBundle(None, True)
package = XPIManager(open(file_), mode='r', name=file_)
contents = package.package_contents()
# We need to have an install.rdf.
assert 'install.rdf' in contents
# Load up the install.rdf into an RDFParser
install_file = package.read('install.rdf')
install_rdf = RDFParser(err, install_file)
results = typedetection.detect_type(err, install_rdf, package)
assert results == expectation
if not failure:
assert not err.failed()
else:
assert err.failed()
return err
def _test_type(file_, expectation, failure=False):
'Tests a file against the expectations'
err = ErrorBundle(None, True)
package = XPIManager(open(file_), mode='r', name=file_)
contents = package.package_contents()
# We need to have an install.rdf.
assert 'install.rdf' in contents
# Load up the install.rdf into an RDFParser
install_file = package.read('install.rdf')
install_rdf = RDFParser(err, install_file)
results = typedetection.detect_type(err, install_rdf, package)
assert results == expectation
if not failure:
assert not err.failed()
else:
assert err.failed()
return err
def _do_test(path, test, failure=True,
require_install=False, set_type=0,
listed=False, xpi_mode='r'):
package_data = open(path, 'rb')
package = XPIManager(package_data, mode=xpi_mode, name=path)
err = ErrorBundle()
if listed:
err.save_resource('listed', True)
# Populate in the dependencies.
if set_type:
err.detected_type = set_type # Conduit test requires type
if require_install:
if 'install.rdf' in package:
err.save_resource('has_install_rdf', True)
rdf_data = package.read('install.rdf')
install_rdf = RDFParser(err, rdf_data)
err.save_resource('install_rdf', install_rdf)
elif 'manifest.json' in package:
err.save_resource('has_manifest_json', True)
manifest_data = package.read('manifest.json')
manifest_json = ManifestJsonParser(err, manifest_data)
err.save_resource('install_rdf', manifest_json)
populate_chrome_manifest(err, package)
test(err, package)
print err.print_summary(verbose=True)
if failure:
assert err.failed()
else:
assert not err.failed()
return err
def test_package(err,
file_,
name,
expectation=PACKAGE_ANY,
for_appversions=None):
'Begins tests for the package.'
# Load up a new instance of an XPI.
try:
package = XPIManager(file_, mode='r', name=name)
has_package_json = 'package.json' in package
has_manifest_json = 'manifest.json' in package
has_install_rdf = 'install.rdf' in package
# install.rdf? | package.json? | manifest.json? | error | use-file
# Yes | No | No | No | install.rdf
# Yes | Yes | No | No | install.rdf
# Yes | No | Yes | No | install.rdf
# No | No | Yes | No | manifest.json
# No | No | No | Yes | install.rdf
# No | Yes | No | No | package.json
# No | No | Yes | Yes | install.rdf
if has_package_json:
_load_package_json(err, package, expectation)
if has_manifest_json:
_load_manifest_json(err, package, expectation)
if has_install_rdf:
_load_install_rdf(err, package, expectation)
except IOError:
# Die on this one because the file won't open.
err.error(('main', 'test_package', 'unopenable'),
'The XPI could not be opened.')
return
except (BadZipfile, zlib_error):
# Die if the zip file is corrupt.
err.error(('submain', '_load_install_rdf', 'badzipfile'),
error='Corrupt ZIP file',
description='We were unable to decompress the zip file.')
return
if package.extension in assumed_extensions:
assumed_type = assumed_extensions[package.extension]
# Is the user expecting a different package type?
if expectation not in (PACKAGE_ANY, assumed_type):
err.error(('main', 'test_package', 'unexpected_type'),
'Unexpected package type (found theme)')
test_inner_package(err, package, for_appversions)
def test_package(err, file_, name, expectation=PACKAGE_ANY,
for_appversions=None):
"Begins tests for the package."
# Load up a new instance of an XPI.
try:
package = XPIManager(file_, mode="r", name=name)
except:
# Die on this one because the file won't open.
return err.error(("main",
"test_package",
"unopenable"),
"The XPI could not be opened.")
# Test the XPI file for corruption.
if package.test():
return err.error(("main",
"test_package",
"corrupt"),
"XPI package appears to be corrupt.")
if package.extension in assumed_extensions:
assumed_type = assumed_extensions[package.extension]
# Is the user expecting a different package type?
if not expectation in (PACKAGE_ANY, assumed_type):
err.error(("main",
"test_package",
"unexpected_type"),
"Unexpected package type (found theme)")
# Test the install.rdf file to see if we can get the type that way.
has_install_rdf = "install.rdf" in package
if has_install_rdf:
_load_install_rdf(err, package, expectation)
return test_inner_package(err, package, for_appversions)
def _run_test(filename, expectation, should_fail=True):
name = "tests/resources/submain/%s" % filename
pack = open(name)
xpi = XPIManager(pack, mode="r", name=name)
err = ErrorBundle(None, True)
submain._load_install_rdf(err, xpi, expectation)
if should_fail:
assert err.failed()
else:
assert not err.failed()
assert err.get_resource("install_rdf")
return err
def test_comparer():
"Tests the function that compares two packages."
ref = XPIManager("tests/resources/l10n/langpack/reference.jar")
ref.locale_name = "en-US"
extra_ref = XPIManager(
"tests/resources/l10n/langpack/extra_files_ref.jar")
pass_ = XPIManager("tests/resources/l10n/langpack/pass.jar")
pass_.locale_name = "en-US"
mfile = XPIManager(
"tests/resources/l10n/langpack/missing_file.jar")
mfile.locale_name = "en-US"
extra = XPIManager(
"tests/resources/l10n/langpack/extra_files.jar")
extra.locale_name = "en-US"
mfileent = XPIManager(
"tests/resources/l10n/langpack/missing_file_entities.jar")
mfileent.locale_name = "en-US"
ment = XPIManager(
"tests/resources/l10n/langpack/missing_entities.jar")
ment.locale_name = "en-US"
assert _compare_packs(ref, pass_) == 3
assert _compare_packs(extra_ref, pass_) == 3
assert _compare_packs(ref, extra) == 3
assert _compare_packs(ref, mfile) == 4
assert _compare_packs(ref, mfileent) == 3
assert _compare_packs(ref, ref) > 3
def test_read_file():
"""Test that a file can be read from the package."""
z = XPIManager(get_path('xpi/install_rdf_only.xpi'))
assert z.read('install.rdf') is not None
def test_valid_name():
"Test that the manager can retrieve the correct file name."
z = XPIManager(get_path('xpi/install_rdf_only.xpi'))
contents = z.package_contents()
assert 'install.rdf' in contents
def test_lp_xpi(err, xpi_package):
"Tests a language pack for L10n completeness"
# Don't even both with the test(s) if there's no chrome.manifest.
if "chrome.manifest" not in xpi_package:
return None
locales = _get_locales(err)
# Get the reference packages.
references = []
support_references = err.get_resource("supports")
if not support_references:
references.append("firefox")
err.info(("testcases_l10ncompleteness", "test_lp_xpi",
"missing_app_support"),
"Supported app missing in localization completeness.",
"While testing in localization comleteness, a list of "
"supported applications for the language pack was not found. "
"This is likely because there are no listed "
"<em:targetApplication> elements in the install.rdf file.")
else:
for support in support_references:
ref_xpi = XPIManager(
os.path.join(os.path.dirname(__file__),
"langpacks/%s.xpi" % support))
ref_xpi.app_name = support
reference_locales = _get_locales(None, ref_xpi)
references.append((ref_xpi, reference_locales))
# Iterate each supported reference package
for (ref_xpi, ref_locales) in references:
# Iterate each locale in each supported reference package
ref_pack = _get_locale_manager(err,
ref_xpi, {
"path": "en-US.jar",
"jarred": True
},
no_cache=True)
for ref_locale_name in ref_locales:
ref_locale = ref_locales[ref_locale_name]
ref_predicate = ref_locale["predicate"]
corresp_locales = [
locales[name] for name in locales
if locales[name]["predicate"] == ref_predicate
]
# If we found no matching locale, then it's missing from the pack
if not corresp_locales:
err.warning(
("testcases_l10ncompleteness", "test_lp_xpi",
"find_corresponding_locale"),
"Could not find corresponding locale", [
"A locale was found in the reference package, "
"however it was not found in the target package.",
"Missing locale: %s" % ref_predicate
],
filename="chrome.manifest")
continue
target_locale = corresp_locales[0]
target_pack = _get_locale_manager(err, xpi_package, target_locale)
if target_pack is None:
continue
results = _compare_packages(reference=ref_pack,
target=target_pack,
ref_base=ref_locale["target"],
locale_base=target_locale["target"])
# Report the findings after each supported app's locale
_aggregate_results(err, results, target_locale)
# Clear the cache at the end of the test
LOCALE_CACHE = {}
def test_lp_xpi(err, xpi_package):
"Tests a language pack for L10n completeness"
# Don't even both with the test(s) if there's no chrome.manifest.
if "chrome.manifest" not in xpi_package:
return None
locales = _get_locales(err)
# Get the reference packages.
references = []
support_references = err.get_resource("supports")
if not support_references:
references.append("firefox")
err.info(
("testcases_l10ncompleteness", "test_lp_xpi", "missing_app_support"),
"Supported app missing in localization completeness.",
"While testing in localization comleteness, a list of "
"supported applications for the language pack was not found. "
"This is likely because there are no listed "
"<em:targetApplication> elements in the install.rdf file.",
)
else:
for support in support_references:
ref_xpi = XPIManager(os.path.join(os.path.dirname(__file__), "langpacks/%s.xpi" % support))
ref_xpi.app_name = support
reference_locales = _get_locales(None, ref_xpi)
references.append((ref_xpi, reference_locales))
# Iterate each supported reference package
for (ref_xpi, ref_locales) in references:
# Iterate each locale in each supported reference package
ref_pack = _get_locale_manager(err, ref_xpi, {"path": "en-US.jar", "jarred": True}, no_cache=True)
for ref_locale_name in ref_locales:
ref_locale = ref_locales[ref_locale_name]
ref_predicate = ref_locale["predicate"]
corresp_locales = [locales[name] for name in locales if locales[name]["predicate"] == ref_predicate]
# If we found no matching locale, then it's missing from the pack
if not corresp_locales:
err.warning(
("testcases_l10ncompleteness", "test_lp_xpi", "find_corresponding_locale"),
"Could not find corresponding locale",
[
"A locale was found in the reference package, "
"however it was not found in the target package.",
"Missing locale: %s" % ref_predicate,
],
filename="chrome.manifest",
)
continue
target_locale = corresp_locales[0]
target_pack = _get_locale_manager(err, xpi_package, target_locale)
if target_pack is None:
continue
results = _compare_packages(
reference=ref_pack,
target=target_pack,
ref_base=ref_locale["target"],
locale_base=target_locale["target"],
)
# Report the findings after each supported app's locale
_aggregate_results(err, results, target_locale)
# Clear the cache at the end of the test
LOCALE_CACHE = {}
def test_get_list():
"Test that the manager can read the file listing"
z = XPIManager("tests/resources/xpi/install_rdf_only.xpi")
assert z.get_file_data()
def _process_file(err,
xpi_package,
name,
file_data,
name_lower,
pollutable=False):
"""Process a single file's content tests."""
extension = os.path.splitext(name_lower)[1]
# If that item is a container file, unzip it and scan it.
if extension == '.jar':
# This is either a subpackage or a nested theme.
is_subpackage = not err.get_resource('is_multipackage')
# Unpack the package and load it up.
package = StringIO(file_data)
try:
sub_xpi = XPIManager(package,
mode='r',
name=name,
subpackage=is_subpackage)
except BadZipfile:
err.error(('testcases_content', 'test_packed_packages',
'jar_subpackage_corrupt'), 'Subpackage corrupt.',
'The subpackage appears to be corrupt, and could not '
'be opened.', name)
return
# Let the error bunder know we're in a sub-package.
err.push_state(name)
err.detected_type = (PACKAGE_SUBPACKAGE
if is_subpackage else PACKAGE_THEME)
err.set_tier(1)
supported_versions = (err.supported_versions.copy()
if err.supported_versions else
err.supported_versions)
if is_subpackage:
testendpoint_validator.test_inner_package(err, sub_xpi)
else:
testendpoint_validator.test_package(err, package, name)
err.pop_state()
err.set_tier(2)
err.supported_versions = supported_versions
elif extension == '.xpi':
# It's not a subpackage, it's a nested extension. These are
# found in multi-extension packages.
# Unpack!
package = StringIO(file_data)
err.push_state(name_lower)
err.set_tier(1)
# There are no expected types for packages within a multi-
# item package.
testendpoint_validator.test_package(err, package, name)
err.pop_state()
err.set_tier(2) # Reset to the current tier
else:
if not file_data:
return
# Convert the file data to unicode.
file_data = unicodehelper.decode(file_data)
if extension in ('.js', '.jsm'):
testendpoint_js.test_js_file(err,
name,
file_data,
pollutable=pollutable)
elif extension == '.css':
testendpoint_css.test_css_file(err, name, file_data)
run_regex_tests(file_data, err, filename=name)
def test_package(err,
file_,
name,
expectation=PACKAGE_ANY,
for_appversions=None):
'Begins tests for the package.'
# Load up a new instance of an XPI.
try:
package = XPIManager(file_, mode='r', name=name)
has_package_json = 'package.json' in package
has_manifest_json = 'manifest.json' in package
has_install_rdf = 'install.rdf' in package
# install.rdf? | package.json? | manifest.json? | error | use-file
# Yes | No | No | No | install.rdf
# Yes | Yes | No | No | install.rdf
# Yes | No | Yes | No | install.rdf
# No | No | Yes | No | manifest.json
# No | No | No | Yes | install.rdf
# No | Yes | No | No | package.json
# No | No | Yes | Yes | install.rdf
if has_package_json:
_load_package_json(err, package, expectation)
if has_manifest_json:
_load_manifest_json(err, package, expectation)
if has_install_rdf:
_load_install_rdf(err, package, expectation)
except IOError:
# Die on this one because the file won't open.
err.error(('main', 'test_package', 'unopenable'),
'The XPI could not be opened.')
return
except (BadZipfile, zlib_error):
# Die if the zip file is corrupt.
err.error(('submain', '_load_install_rdf', 'badzipfile'),
error='Corrupt ZIP file',
description='We were unable to decompress the zip file.')
return
if package.extension in assumed_extensions:
assumed_type = assumed_extensions[package.extension]
# Is the user expecting a different package type?
if expectation not in (PACKAGE_ANY, assumed_type):
err.error(('main', 'test_package', 'unexpected_type'),
'Unexpected package type (found theme)')
if (err.get_resource('has_manifest_json')
and not err.get_resource('has_package_json')
and not err.get_resource('has_install_rdf')):
# It's a WebExtension. Those are not supported by amo-validator, we
# should be using the linter instead. Only reason we could be using
# amo validator with a WebExtension is if a developer tried to use
# the compatibilty checker with a WebExtension, so add a relevant error
# message.
format_args = {'link': MDN_DOC % 'Mozilla/Add-ons/WebExtensions'}
err.error(
('submain', 'test_inner_package', 'webextension'),
'This tool only works with legacy add-ons. See {link} for more '
'information about WebExtension APIs.'.format(**format_args))
else:
test_inner_package(err, package, for_appversions)
def packager(data, xpi_path, features):
"""Package an add-on from input data. The resulting package will be
saved as xpi_path.
data format:
- id : <em:id> value
- version : <em:version>
- name : <em:name>
- description : <em:description>
- author_name : <em:author>
- contributors : \n-delimited list of contributors
- targetapplications : Dict in the form of:
{
"min_ver": "3.6",
"max_ver": "6.0a1",
"guid": "...",
}
- uuid : A UUID value that is unique to this package
- slug : A slug value based on the name which will be used as a
package identifier.
xpi_path should be the file path to build the XPI at.
features should be a set containing string names of each of the
features to include.
"""
# Sanitize the slug.
data['slug'] = _slugify(data.get('slug', ''))
# Instantiate the XPI Manager.
from validator.xpi import XPIManager
xpi = XPIManager(xpi_path, mode='w')
xpi.write('install.rdf', build_installrdf(data, features))
xpi.write('chrome.manifest', build_chrome_manifest(data, features))
# Sanitize all the input after building `install.rdf` (which is escaped
# when jinja renders the template) to prevent doubly escaping the input.
data = escape_all(data)
_write_resource('defaults/preferences/prefs.js', xpi, data)
_write_resource('chrome/skin/overlay.css', xpi, data)
_write_resource('chrome/locale/en-US/overlay.dtd', xpi, data)
_write_resource('chrome/locale/en-US/overlay.properties', xpi, data)
if 'about_dialog' in features:
_write_resource('chrome/content/about.xul', xpi, data)
_write_resource('chrome/locale/en-US/about.dtd', xpi)
if 'preferences_dialog' in features:
_write_resource('chrome/content/options.xul', xpi, data)
_write_resource('chrome/locale/en-US/options.dtd', xpi, data)
if 'toolbar_button' in features:
_write_resource('chrome/skin/toolbar-button.png', xpi)
if 'sidebar_support' in features:
_write_resource('chrome/content/ff-sidebar.js', xpi)
_write_resource('chrome/content/ff-sidebar.xul', xpi, data)
xpi.write('chrome/content/ff-overlay.xul',
build_ffoverlay_xul(data, features))
_write_resource('chrome/content/ff-overlay.js', xpi, data)
xpi.zf.close()
return xpi_path
def test_comparer():
'Tests the function that compares two packages.'
ref = XPIManager('tests/resources/l10n/langpack/reference.jar')
ref.locale_name = 'en-US'
extra_ref = XPIManager('tests/resources/l10n/langpack/extra_files_ref.jar')
pass_ = XPIManager('tests/resources/l10n/langpack/pass.jar')
pass_.locale_name = 'en-US'
mfile = XPIManager('tests/resources/l10n/langpack/missing_file.jar')
mfile.locale_name = 'en-US'
extra = XPIManager('tests/resources/l10n/langpack/extra_files.jar')
extra.locale_name = 'en-US'
mfileent = XPIManager(
'tests/resources/l10n/langpack/missing_file_entities.jar')
mfileent.locale_name = 'en-US'
ment = XPIManager('tests/resources/l10n/langpack/missing_entities.jar')
ment.locale_name = 'en-US'
assert _compare_packs(ref, pass_) == 3
assert _compare_packs(extra_ref, pass_) == 3
assert _compare_packs(ref, extra) == 3
assert _compare_packs(ref, mfile) == 4
assert _compare_packs(ref, mfileent) == 3
assert _compare_packs(ref, ref) > 3
def test_read_file():
"Test that a file can be read from the package"
z = XPIManager("tests/resources/xpi/install_rdf_only.xpi")
assert z.read("install.rdf") is not None
def test_valid_name():
"Test that the manager can retrieve the correct file name"
z = XPIManager("tests/resources/xpi/install_rdf_only.xpi")
contents = z.get_file_data()
assert "install.rdf" in contents
assert z.test() == False
def test_get_list():
"Test that the manager can read the file listing"
z = XPIManager("tests/resources/xpi/install_rdf_only.xpi")
assert z.package_contents()
def test_read_file():
"""Test that a file can be read from the package."""
z = XPIManager(get_path('xpi/install_rdf_only.xpi'))
assert z.read('install.rdf') is not None
def __init__(self, filename, project=None, release=None, name=None):
"""
Fills in a list of locales from the chrome.manifest file.
"""
Bundle.__init__(self, project, release)
self.xpi = XPIManager(filename, name=name)
# here we will store managers for jarfiles
self.jarfiles = {}
chrome = ChromeManifest(self.xpi.read("chrome.manifest"), "manifest")
locales = list(chrome.get_triples("locale"))
if not locales:
return None
# read the list
for locale in locales:
code, location = locale["object"].split()
# finding out the language of the locale
try:
lang = self._get_lang(code)
except Language.DoesNotExist:
self.log("Locale %s SKIPPED" % code, "font-weight:bold")
continue
# Locales can be bundled in JARs
jarred = location.startswith("jar:")
if jarred:
# We just care about the JAR path
location = location[4:]
split_location = location.split("!", 2)
# Ignore malformed JAR URIs.
if len(split_location) < 2:
continue
jarname, location = split_location
# missing file mentioned
if jarname not in self.xpi:
continue
# may be we have already read this one
if jarname in self.jarfiles:
package = self.jarfiles[jarname]
else:
jar = StringIO(self.xpi.read(jarname))
package = XPIManager(jar, mode="r", name=jarname)
else:
package = self.xpi
# and now we read files from there
location = location.strip('/')
result = {}
for f in package.package_contents():
f = f.strip("/")
if f.startswith(location) and f != location:
result[f.split("/")[-1]] = package.read(f)
# file with same name in different jars can get overwritten
if lang not in self.locales:
self.locales[lang] = result
else:
self.locales[lang].update(result)
def test_open():
"""Test that the manager will open the package."""
z = XPIManager(get_path('xpi/install_rdf_only.xpi'))
assert z is not None
def __init__(self, filename, project=None, release=None, name=None):
"""
Fills in a list of locales from the chrome.manifest file.
"""
Bundle.__init__(self, project, release)
self.xpi = XPIManager(filename, name=name)
# here we will store managers for jarfiles
self.jarfiles = {}
chrome = ChromeManifest(self.xpi.read("chrome.manifest"), "manifest")
locales = list(chrome.get_triples("locale"))
if not locales:
return None
# read the list
for locale in locales:
code, location = locale["object"].split()
# finding out the language of the locale
try:
lang = self._get_lang(code)
except Language.DoesNotExist:
self.log("Locale %s SKIPPED" % code, "font-weight:bold")
continue
# Locales can be bundled in JARs
jarred = location.startswith("jar:")
if jarred:
# We just care about the JAR path
location = location[4:]
split_location = location.split("!", 2)
# Ignore malformed JAR URIs.
if len(split_location) < 2:
continue
jarname, location = split_location
# missing file mentioned
if jarname not in self.xpi:
continue
# may be we have already read this one
if jarname in self.jarfiles:
package = self.jarfiles[jarname]
else:
jar = StringIO(self.xpi.read(jarname))
package = XPIManager(jar, mode="r", name=jarname)
else:
package = self.xpi
# and now we read files from there
location = location.strip('/')
result = {}
for f in package.package_contents():
f = f.strip("/")
if f.startswith(location) and f != location:
result[f.split("/")[-1]] = package.read(f)
# file with same name in different jars can get overwritten
if lang not in self.locales:
self.locales[lang] = result
else:
self.locales[lang].update(result)
def test_valid_name():
'Test that the manager can retrieve the correct file name.'
z = XPIManager(get_path('xpi/install_rdf_only.xpi'))
contents = z.package_contents()
assert 'install.rdf' in contents
def test_valid_name():
"Test that the manager can retrieve the correct file name."
z = XPIManager(get_path("xpi/install_rdf_only.xpi"))
contents = z.package_contents()
assert "install.rdf" in contents
assert z.test() == False
def test_packed_packages(err, package_contents=None, xpi_package=None):
"Tests XPI and JAR files for naughty content."
processed_files = 0
hash_whitelist = [x[:-1] for x in
open(os.path.join(os.path.dirname(__file__),
'whitelist_hashes.txt')).readlines()]
# Iterate each item in the package.
for name, data in package_contents.items():
if name.startswith("__MACOSX") or \
name.startswith(".DS_Store"):
continue
if name.split("/")[-1].startswith("._"):
err.notice(("testcases_content",
"test_packed_packages",
"macintosh_junk"),
"Garbage file found.",
["""A junk file has been detected. It may cause
problems with proper operation of the add-on down the
road.""",
"It is recommended that you delete the file"],
name)
try:
file_data = xpi_package.read(name)
except KeyError: # pragma: no cover
_read_error(err, name)
# Skip over whitelisted hashes
hash = hashlib.sha1(file_data).hexdigest()
if hash in hash_whitelist:
continue
processed = False
# If that item is a container file, unzip it and scan it.
if data["extension"] == "jar":
# This is either a subpackage or a nested theme.
# Whether this is a subpackage or a nested theme is
# determined by whether it is in the root folder or not.
# Subpackages are always found in a directory such as
# /chrome or /content.
is_subpackage = name.count("/") > 0
# Unpack the package and load it up.
package = StringIO(file_data)
sub_xpi = XPIManager(package, name, is_subpackage)
if not sub_xpi.zf:
err.error(("testcases_content",
"test_packed_packages",
"jar_subpackage_corrupt"),
"Subpackage corrupt.",
"""The subpackage could not be opened due to
issues with corruption. Ensure that the file
is valid.""",
name)
continue
temp_contents = sub_xpi.get_file_data()
# Let the error bunder know we're in a sub-package.
err.push_state(data["name_lower"])
err.set_type(PACKAGE_SUBPACKAGE) # Subpackage
testendpoint_validator.test_inner_package(err,
temp_contents,
sub_xpi)
err.tier = 2
package.close()
err.pop_state()
elif data["extension"] == "xpi":
# It's not a subpackage, it's a nested extension. These are
# found in multi-extension packages.
# Unpack!
package = StringIO(file_data)
err.push_state(data["name_lower"])
# There are no expected types for packages within a multi-
# item package.
testendpoint_validator.test_package(err, package, name)
err.tier = 2 # Reset to the current tier
package.close()
err.pop_state()
elif data["extension"] in ("xul", "xml", "html", "xhtml"):
parser = testendpoint_markup.MarkupParser(err)
parser.process(name,
charsethelper.decode(file_data),
data["extension"])
processed = True
elif data["extension"] in ("css", "js", "jsm"):
if not file_data:
continue
file_data = charsethelper.decode(file_data)
if data["extension"] == "css":
testendpoint_css.test_css_file(err,
name,
file_data)
elif data["extension"] in ("js", "jsm"):
testendpoint_js.test_js_file(err,
name,
file_data)
# This is tested in test_langpack.py
if err.detected_type == PACKAGE_LANGPACK and not processed:
testendpoint_langpack.test_unsafe_html(err,
name,
file_data)
# This aids in creating unit tests.
processed_files += 1
return processed_files
def test_lp_xpi(err, xpi_package):
'Tests a language pack for L10n completeness'
# Don't even both with the test(s) if there's no chrome.manifest.
if 'chrome.manifest' not in xpi_package:
return None
locales = _get_locales(err)
# Get the reference packages.
references = []
support_references = err.get_resource('supports')
if not support_references:
references.append('firefox')
err.info(('testcases_l10ncompleteness',
'test_lp_xpi',
'missing_app_support'),
'Supported app missing in localization completeness.',
'While testing in localization comleteness, a list of '
'supported applications for the language pack was not found. '
'This is likely because there are no listed '
'<em:targetApplication> elements in the install.rdf file.')
else:
for support in support_references:
ref_xpi = XPIManager(os.path.join(os.path.dirname(__file__),
'langpacks/%s.xpi' % support))
ref_xpi.app_name = support
reference_locales = _get_locales(None, ref_xpi)
references.append((ref_xpi, reference_locales))
# Iterate each supported reference package
for (ref_xpi, ref_locales) in references:
# Iterate each locale in each supported reference package
ref_pack = _get_locale_manager(err,
ref_xpi,
{'path': 'en-US.jar',
'jarred': True},
no_cache=True)
for ref_locale_name in ref_locales:
ref_locale = ref_locales[ref_locale_name]
ref_predicate = ref_locale['predicate']
corresp_locales = [locales[name] for name
in locales
if locales[name]['predicate'] == ref_predicate]
# If we found no matching locale, then it's missing from the pack
if not corresp_locales:
err.warning(('testcases_l10ncompleteness',
'test_lp_xpi',
'find_corresponding_locale'),
'Could not find corresponding locale',
['A locale was found in the reference package, '
'however it was not found in the target package.',
'Missing locale: %s' % ref_predicate],
filename='chrome.manifest')
continue
target_locale = corresp_locales[0]
target_pack = _get_locale_manager(err,
xpi_package,
target_locale)
if target_pack is None:
continue
results = _compare_packages(reference=ref_pack,
target=target_pack,
ref_base=ref_locale['target'],
locale_base=target_locale['target'])
# Report the findings after each supported app's locale
_aggregate_results(err, results, target_locale)
# Clear the cache at the end of the test
LOCALE_CACHE.clear()
def _process_file(err,
xpi_package,
name,
file_data,
name_lower,
pollutable=False):
"""Process a single file's content tests."""
# If that item is a container file, unzip it and scan it.
if name_lower.endswith(".jar"):
# This is either a subpackage or a nested theme.
is_subpackage = not err.get_resource("is_multipackage")
# Unpack the package and load it up.
package = StringIO(file_data)
try:
sub_xpi = XPIManager(package,
mode="r",
name=name,
subpackage=is_subpackage)
except Exception:
err.error(("testcases_content", "test_packed_packages",
"jar_subpackage_corrupt"), "Subpackage corrupt.",
"The subpackage could not be opened due to issues "
"with corruption. Ensure that the file is valid.", name)
return None
# Let the error bunder know we're in a sub-package.
err.push_state(name)
err.detected_type = (PACKAGE_SUBPACKAGE
if is_subpackage else PACKAGE_THEME)
err.set_tier(1)
supported_versions = (err.supported_versions.copy()
if err.supported_versions else
err.supported_versions)
if is_subpackage:
testendpoint_validator.test_inner_package(err, sub_xpi)
else:
testendpoint_validator.test_package(err, package, name)
err.pop_state()
err.set_tier(2)
err.supported_versions = supported_versions
elif name_lower.endswith(".xpi"):
# It's not a subpackage, it's a nested extension. These are
# found in multi-extension packages.
# Unpack!
package = StringIO(file_data)
err.push_state(name_lower)
err.set_tier(1)
# There are no expected types for packages within a multi-
# item package.
testendpoint_validator.test_package(err, package, name)
err.pop_state()
err.set_tier(2) # Reset to the current tier
elif name_lower.endswith((".css", ".js", ".jsm")):
if not file_data:
return None
# Convert the file data to unicode
file_data = unicodehelper.decode(file_data)
is_js = False
if name_lower.endswith(".css"):
testendpoint_css.test_css_file(err, name, file_data)
elif name_lower.endswith((".js", ".jsm")):
is_js = True
testendpoint_js.test_js_file(err,
name,
file_data,
pollutable=pollutable)
run_regex_tests(file_data, err, name, is_js=is_js)
return True
return False
def test_lp_xpi(err, xpi_package):
'Tests a language pack for L10n completeness'
# Don't even both with the test(s) if there's no chrome.manifest.
if 'chrome.manifest' not in xpi_package:
return None
locales = _get_locales(err)
# Get the reference packages.
references = []
support_references = err.get_resource('supports')
if not support_references:
references.append('firefox')
err.info(('testcases_l10ncompleteness', 'test_lp_xpi',
'missing_app_support'),
'Supported app missing in localization completeness.',
'While testing in localization comleteness, a list of '
'supported applications for the language pack was not found. '
'This is likely because there are no listed '
'<em:targetApplication> elements in the install.rdf file.')
else:
for support in support_references:
ref_xpi = XPIManager(
os.path.join(os.path.dirname(__file__),
'langpacks/%s.xpi' % support))
ref_xpi.app_name = support
reference_locales = _get_locales(None, ref_xpi)
references.append((ref_xpi, reference_locales))
# Iterate each supported reference package
for (ref_xpi, ref_locales) in references:
# Iterate each locale in each supported reference package
ref_pack = _get_locale_manager(err,
ref_xpi, {
'path': 'en-US.jar',
'jarred': True
},
no_cache=True)
for ref_locale_name in ref_locales:
ref_locale = ref_locales[ref_locale_name]
ref_predicate = ref_locale['predicate']
corresp_locales = [
locales[name] for name in locales
if locales[name]['predicate'] == ref_predicate
]
# If we found no matching locale, then it's missing from the pack
if not corresp_locales:
err.warning(
('testcases_l10ncompleteness', 'test_lp_xpi',
'find_corresponding_locale'),
'Could not find corresponding locale', [
'A locale was found in the reference package, '
'however it was not found in the target package.',
'Missing locale: %s' % ref_predicate
],
filename='chrome.manifest')
continue
target_locale = corresp_locales[0]
target_pack = _get_locale_manager(err, xpi_package, target_locale)
if target_pack is None:
continue
results = _compare_packages(reference=ref_pack,
target=target_pack,
ref_base=ref_locale['target'],
locale_base=target_locale['target'])
# Report the findings after each supported app's locale
_aggregate_results(err, results, target_locale)
# Clear the cache at the end of the test
LOCALE_CACHE.clear()
