def test_mozilla_trademark_for_prefix_allowed(self, resolve_message):
resolve_message.return_value = 'Notify for Mozilla'
addon = amo.tests.addon_factory()
file_obj = addon.current_version.file
fixture = 'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi'
with amo.tests.copy_file(fixture, file_obj.file_path):
utils.parse_xpi(file_obj.file_path)
def test_parse_xpi_no_manifest(self):
fake_zip = utils.make_xpi({'dummy': 'dummy'})
with mock.patch(
'olympia.files.utils.get_file'
) as get_file_mock, self.assertRaises(utils.NoManifestFound) as exc:
get_file_mock.return_value = fake_zip
utils.parse_xpi(None)
assert isinstance(exc.exception, forms.ValidationError)
assert exc.exception.message == ('No manifest.json found')
def test_mozilla_trademark_for_prefix_allowed(self, resolve_message):
resolve_message.return_value = 'Notify for Mozilla'
addon = amo.tests.addon_factory()
file_obj = addon.current_version.all_files[0]
fixture = (
'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi')
with amo.tests.copy_file(fixture, file_obj.file_path):
utils.parse_xpi(file_obj.file_path)
def test_mozilla_trademark_for_prefix_not_allowed(self, resolve_message):
resolve_message.return_value = 'Notify for Mozilla'
addon = amo.tests.addon_factory()
file_obj = addon.current_version.all_files[0]
fixture = (
'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi')
with amo.tests.copy_file(fixture, file_obj.file_path):
with self.assertRaises(forms.ValidationError):
utils.parse_xpi(file_obj.file_path)
def test_mozilla_trademark_disallowed(self, resolve_message):
resolve_message.return_value = 'Notify Mozilla'
addon = amo.tests.addon_factory()
file_obj = addon.current_version.file
fixture = 'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi'
with amo.tests.copy_file(fixture, file_obj.file_path):
with pytest.raises(forms.ValidationError) as exc:
utils.parse_xpi(file_obj.file_path)
assert dict(exc.value.messages)['en-us'].startswith(
'Add-on names cannot contain the Mozilla or')
def test_mozilla_trademark_disallowed(self, resolve_message):
resolve_message.return_value = 'Notify Mozilla'
addon = amo.tests.addon_factory()
file_obj = addon.current_version.all_files[0]
fixture = (
'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi')
with amo.tests.copy_file(fixture, file_obj.file_path):
with pytest.raises(forms.ValidationError) as exc:
utils.parse_xpi(file_obj.file_path)
assert dict(exc.value.messages)['en-us'].startswith(
u'Add-on names cannot contain the Mozilla or'
)
def extract_optional_permissions(ids, **kw):
log.info(
'[%s@%s] Extracting permissions from Files, from id: %s to id: %s...' %
(len(ids), extract_optional_permissions.rate_limit, ids[0], ids[-1]))
files = File.objects.filter(pk__in=ids).no_transforms()
# A user needs to be passed down to parse_xpi(), so we use the task user.
user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
for file_ in files:
try:
log.info('Parsing File.id: %s @ %s' %
(file_.pk, file_.current_file_path))
parsed_data = parse_xpi(file_.current_file_path,
addon=file_.addon,
user=user)
optional_permissions = parsed_data.get('optional_permissions', [])
if optional_permissions:
log.info('Found %s optional permissions for: %s' %
(len(optional_permissions), file_.pk))
WebextPermission.objects.update_or_create(
defaults={'optional_permissions': optional_permissions},
file=file_)
except Exception as err:
log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def test_bump_version_in_manifest_json(file_obj):
with amo.tests.copy_file(
'src/olympia/files/fixtures/files/webextension.xpi',
file_obj.file_path):
utils.update_version_number(file_obj, '0.0.1.1-signed')
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['version'] == '0.0.1.1-signed'
def test_bump_version_in_package_json(file_obj):
with amo.tests.copy_file(
'src/olympia/files/fixtures/files/new-format-0.0.1.xpi',
file_obj.file_path):
utils.update_version_number(file_obj, '0.0.1.1-signed')
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['version'] == '0.0.1.1-signed'
def extract_webext_permissions(ids, **kw):
log.info(
'[%s@%s] Extracting permissions from Files, starting at id: %s...' %
(len(ids), extract_webext_permissions.rate_limit, ids[0]))
files = File.objects.filter(pk__in=ids).no_transforms()
# A user needs to be passed down to parse_xpi(), so we use the task user.
user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
for file_ in files:
try:
log.info('Parsing File.id: %s @ %s' %
(file_.pk, file_.current_file_path))
parsed_data = parse_xpi(file_.current_file_path, user=user)
permissions = parsed_data.get('permissions', [])
# Add content_scripts host matches too.
for script in parsed_data.get('content_scripts', []):
permissions.extend(script.get('matches', []))
if permissions:
log.info('Found %s permissions for: %s' %
(len(permissions), file_.pk))
WebextPermission.objects.update_or_create(
defaults={'permissions': permissions}, file=file_)
except Exception as err:
log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def extract_theme_properties(addon, channel):
version = addon.find_latest_version(channel)
if not version or not version.all_files:
return {}
try:
parsed_data = parse_xpi(
version.all_files[0].file_path, addon=addon, user=core.get_user())
except ValidationError:
# If we can't parse the existing manifest safely return.
return {}
theme_props = parsed_data.get('theme', {})
# pre-process colors to convert chrome style colors and strip spaces
theme_props['colors'] = dict(
process_color_value(prop, color)
for prop, color in theme_props.get('colors', {}).items())
# replace headerURL with path to existing background
if 'images' in theme_props:
if 'theme_frame' in theme_props['images']:
header_url = theme_props['images'].pop('theme_frame')
if 'headerURL' in theme_props['images']:
header_url = theme_props['images'].pop('headerURL')
if header_url:
theme_props['images']['headerURL'] = '/'.join((
user_media_url('addons'), text_type(addon.id),
text_type(version.id), header_url))
return theme_props
def test_bump_version_in_manifest_json(file_obj):
create_switch('webextensions')
with amo.tests.copy_file(
'src/olympia/files/fixtures/files/webextension.xpi',
file_obj.file_path):
utils.update_version_number(file_obj, '0.0.1.1-signed')
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['version'] == '0.0.1.1-signed'
def test_moz_signed_extension_no_strict_compat(self):
addon = amo.tests.addon_factory()
file_obj = addon.current_version.all_files[0]
file_obj.update(is_mozilla_signed_extension=True)
fixture = ('src/olympia/files/fixtures/files/'
'legacy-addon-already-signed-0.1.0.xpi')
with amo.tests.copy_file(fixture, file_obj.file_path):
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['is_mozilla_signed_extension']
assert not parsed['strict_compatibility']
def test_moz_signed_extension_no_strict_compat(self):
addon = amo.tests.addon_factory()
file_obj = addon.current_version.all_files[0]
file_obj.update(is_mozilla_signed_extension=True)
fixture = (
'src/olympia/files/fixtures/files/'
'legacy-addon-already-signed-0.1.0.xpi')
with amo.tests.copy_file(fixture, file_obj.file_path):
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['is_mozilla_signed_extension']
assert not parsed['strict_compatibility']
def test_moz_signed_extension_reuse_strict_compat(self):
addon = amo.tests.addon_factory()
user = amo.tests.user_factory(email='*****@*****.**')
file_obj = addon.current_version.all_files[0]
file_obj.update(is_mozilla_signed_extension=True)
fixture = ('src/olympia/files/fixtures/files/'
'legacy-addon-already-signed-strict-compat-0.1.0.xpi')
with amo.tests.copy_file(fixture, file_obj.file_path):
parsed = utils.parse_xpi(file_obj.file_path, user=user)
assert parsed['is_mozilla_signed_extension']
# We set `strictCompatibility` in install.rdf
assert parsed['strict_compatibility']
def test_moz_signed_extension_reuse_strict_compat(self):
addon = amo.tests.addon_factory()
user = amo.tests.user_factory(email='*****@*****.**')
file_obj = addon.current_version.all_files[0]
file_obj.update(is_mozilla_signed_extension=True)
fixture = (
'src/olympia/files/fixtures/files/'
'legacy-addon-already-signed-strict-compat-0.1.0.xpi')
with amo.tests.copy_file(fixture, file_obj.file_path):
parsed = utils.parse_xpi(file_obj.file_path, user=user)
assert parsed['is_mozilla_signed_extension']
# We set `strictCompatibility` in install.rdf
assert parsed['strict_compatibility']
def test_bump_version_in_manifest_json(file_obj):
AppVersion.objects.create(application=amo.FIREFOX.id,
version=amo.DEFAULT_WEBEXT_MIN_VERSION)
AppVersion.objects.create(application=amo.FIREFOX.id,
version=amo.DEFAULT_WEBEXT_MAX_VERSION)
AppVersion.objects.create(application=amo.ANDROID.id,
version=amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID)
AppVersion.objects.create(application=amo.ANDROID.id,
version=amo.DEFAULT_WEBEXT_MAX_VERSION)
with amo.tests.copy_file(
'src/olympia/files/fixtures/files/webextension.xpi',
file_obj.file_path):
utils.update_version_number(file_obj, '0.0.1.1-signed')
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['version'] == '0.0.1.1-signed'
def test_bump_version_in_manifest_json(file_obj):
AppVersion.objects.create(application=amo.FIREFOX.id,
version=amo.DEFAULT_WEBEXT_MIN_VERSION)
AppVersion.objects.create(application=amo.FIREFOX.id,
version=amo.DEFAULT_WEBEXT_MAX_VERSION)
AppVersion.objects.create(application=amo.ANDROID.id,
version=amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID)
AppVersion.objects.create(application=amo.ANDROID.id,
version=amo.DEFAULT_WEBEXT_MAX_VERSION)
with amo.tests.copy_file(
'src/olympia/files/fixtures/files/webextension.xpi',
file_obj.file_path):
utils.update_version_number(file_obj, '0.0.1.1-signed')
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['version'] == '0.0.1.1-signed'
def extract_theme_properties(addon, channel):
version = addon.find_latest_version(channel)
if not version or not version.all_files:
return {}
try:
parsed_data = parse_xpi(
version.all_files[0].file_path, addon=addon, user=core.get_user())
except ValidationError:
# If we can't parse the existing manifest safely return.
return {}
theme_props = parsed_data.get('theme', {})
# pre-process colors to convert chrome style colors and strip spaces
theme_props['colors'] = dict(
process_color_value(prop, color)
for prop, color in theme_props.get('colors', {}).items())
return theme_props
def extract_theme_properties(addon, channel):
version = addon.find_latest_version(channel)
if not version or not version.all_files:
return {}
try:
parsed_data = parse_xpi(
version.all_files[0].file_path, addon=addon, user=core.get_user())
except ValidationError:
# If we can't parse the existing manifest safely return.
return {}
theme_props = parsed_data.get('theme', {})
# pre-process colors to convert chrome style colors and strip spaces
theme_props['colors'] = dict(
process_color_value(prop, color)
for prop, color in theme_props.get('colors', {}).items())
return theme_props
def extract_webext_permissions(ids, **kw):
log.info('[%s@%s] Extracting permissions from Files, starting at id: %s...'
% (len(ids), extract_webext_permissions.rate_limit, ids[0]))
files = File.objects.filter(pk__in=ids).no_transforms()
for file_ in files:
try:
log.info('Parsing File.id: %s @ %s' %
(file_.pk, file_.current_file_path))
parsed_data = parse_xpi(file_.current_file_path, check=False)
permissions = parsed_data.get('permissions')
if permissions:
log.info('Found %s permissions for: %s' %
(len(permissions), file_.pk))
WebextPermission.objects.update_or_create(
defaults={'permissions': permissions}, file=file_)
except Exception, err:
log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def extract_webext_permissions(ids, **kw):
log.info(
'[%s@%s] Extracting permissions from Files, starting at id: %s...' %
(len(ids), extract_webext_permissions.rate_limit, ids[0]))
files = File.objects.filter(pk__in=ids).no_transforms()
for file_ in files:
try:
log.info('Parsing File.id: %s @ %s' %
(file_.pk, file_.current_file_path))
parsed_data = parse_xpi(file_.current_file_path, check=False)
permissions = parsed_data.get('permissions')
if permissions:
log.info('Found %s permissions for: %s' %
(len(permissions), file_.pk))
WebextPermission.objects.update_or_create(
defaults={'permissions': permissions}, file=file_)
except Exception, err:
log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def extract_theme_properties(addon, channel):
version = addon.find_latest_version(channel)
if not version or not version.all_files:
return {}
try:
parsed_data = parse_xpi(
version.all_files[0].file_path, addon=addon, user=core.get_user())
except ValidationError:
# If we can't parse the existing manifest safely return.
return {}
theme_props = parsed_data.get('theme', {})
# pre-process colors to deprecated colors; strip spaces.
theme_props['colors'] = dict(
process_color_value(prop, color)
for prop, color in theme_props.get('colors', {}).items())
# upgrade manifest from deprecated headerURL to theme_frame
if 'headerURL' in theme_props.get('images', {}):
url = theme_props['images'].pop('headerURL')
theme_props['images']['theme_frame'] = url
return theme_props
def populate_e10s_feature_compatibility(ids, **kwargs):
log.info(
'[%s@%s] Populating e10s feature compatibility '
'on addons starting w/ id: %s...' %
(len(ids), populate_e10s_feature_compatibility.rate_limit, ids[0]))
addons = Addon.unfiltered.filter(pk__in=ids)
for addon in addons:
file_ = addon.get_latest_file()
if file_:
try:
parsed = parse_xpi(file_.file_path, addon=addon)
except ValidationError:
log.warn('Could not parse XPI for addon %s, file %s' %
(addon.pk, file_.pk))
continue
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(
addon=addon)[0])
feature_compatibility.update(e10s=parsed['e10s_compatibility'])
def populate_e10s_feature_compatibility(ids, **kwargs):
log.info(
'[%s@%s] Populating e10s feature compatibility '
'on addons starting w/ id: %s...' % (
len(ids), populate_e10s_feature_compatibility.rate_limit, ids[0]))
addons = Addon.unfiltered.filter(pk__in=ids)
for addon in addons:
file_ = addon.get_latest_file()
if file_:
try:
parsed = parse_xpi(file_.file_path, addon=addon)
except ValidationError:
log.warn('Could not parse XPI for addon %s, file %s' % (
addon.pk, file_.pk))
continue
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0]
)
feature_compatibility.update(e10s=parsed['e10s_compatibility'])
def extract_webext_permissions(ids, **kw):
log.info('[%s@%s] Extracting permissions from Files, starting at id: %s...'
% (len(ids), extract_webext_permissions.rate_limit, ids[0]))
files = File.objects.filter(pk__in=ids).no_transforms()
# A user needs to be passed down to parse_xpi(), so we use the task user.
user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
for file_ in files:
try:
log.info('Parsing File.id: %s @ %s' %
(file_.pk, file_.current_file_path))
parsed_data = parse_xpi(file_.current_file_path, user=user)
permissions = parsed_data.get('permissions', [])
# Add content_scripts host matches too.
for script in parsed_data.get('content_scripts', []):
permissions.extend(script.get('matches', []))
if permissions:
log.info('Found %s permissions for: %s' %
(len(permissions), file_.pk))
WebextPermission.objects.update_or_create(
defaults={'permissions': permissions}, file=file_)
except Exception as err:
log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def test_bump_version_in_alt_install_rdf(file_obj):
with amo.tests.copy_file('src/olympia/files/fixtures/files/alt-rdf.xpi',
file_obj.file_path):
utils.update_version_number(file_obj, '2.1.106.1-signed')
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['version'] == '2.1.106.1-signed'
def test_bump_version_in_install_rdf(file_obj):
with amo.tests.copy_file("src/olympia/files/fixtures/files/jetpack.xpi", file_obj.file_path):
utils.update_version_number(file_obj, "1.3.1-signed")
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed["version"] == "1.3.1-signed"
def test_bump_version_in_manifest_json(file_obj):
with amo.tests.copy_file("src/olympia/files/fixtures/files/webextension.xpi", file_obj.file_path):
utils.update_version_number(file_obj, "0.0.1.1-signed")
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed["version"] == "0.0.1.1-signed"
def repack_fileupload(results, upload_pk):
log.info('Starting task to repackage FileUpload %s', upload_pk)
upload = FileUpload.objects.get(pk=upload_pk)
# When a FileUpload is created and a file added to it, if it's a xpi/zip,
# it should be move to upload.path, and it should have a .zip extension,
# so we only need to care about that extension here.
# We don't trust upload.name: it's the original filename as used by the
# developer, so it could be something else.
if upload.path.endswith('.zip'):
timer = StopWatch('files.tasks.repack_fileupload.')
timer.start()
# tempdir must *not* be on TMP_PATH, we want local fs instead. It will be
# deleted automatically once we exit the context manager.
with tempfile.TemporaryDirectory(
prefix='repack_fileupload_extract') as tempdir:
try:
extract_zip(upload.path, tempdir=tempdir)
if waffle.switch_is_active('enable-manifest-normalization'):
manifest = Path(tempdir) / 'manifest.json'
if manifest.exists():
try:
xpi_data = parse_xpi(upload.path, minimal=True)
if not xpi_data.get('is_mozilla_signed_extension',
False):
json_data = ManifestJSONExtractor(
manifest.read_bytes()).data
manifest.write_text(
json.dumps(json_data, indent=2))
except Exception:
# If we cannot normalize the manifest file, we skip
# this step and let the linter catch the exact
# cause in order to return a more appropriate error
# than "unexpected error", which would happen if
# this task was handling the error itself.
pass
except Exception as exc:
# Something bad happened, maybe we couldn't parse the zip file.
# @validation_task should ensure the exception is caught and
# transformed in a generic error message for the developer, so we
# just log it and re-raise.
log.exception('Could not extract upload %s for repack.',
upload_pk,
exc_info=exc)
raise
timer.log_interval('1.extracted')
log.info('Zip from upload %s extracted, repackaging', upload_pk)
# We'll move the file to its final location below with move_stored_file(),
# so don't let tempfile delete it.
file_ = tempfile.NamedTemporaryFile(dir=settings.TMP_PATH,
suffix='.zip',
delete=False)
shutil.make_archive(
os.path.splitext(file_.name)[0], 'zip', tempdir)
with open(file_.name, 'rb') as f:
upload.hash = 'sha256:%s' % get_sha256(f)
timer.log_interval('2.repackaged')
log.info('Zip from upload %s repackaged, moving file back', upload_pk)
storage.move_stored_file(file_.name, upload.path)
timer.log_interval('3.moved')
upload.save()
timer.log_interval('4.end')
else:
log.info('Not repackaging upload %s, it is not a zip file.', upload_pk)
return results
def test_bump_version_in_alt_install_rdf(file_obj):
with amo.tests.copy_file('src/olympia/files/fixtures/files/alt-rdf.xpi',
file_obj.file_path):
utils.update_version_number(file_obj, '2.1.106.1-signed')
parsed = utils.parse_xpi(file_obj.file_path)
assert parsed['version'] == '2.1.106.1-signed'
def test_parse_xpi():
"""Fire.fm can sometimes give us errors. Let's prevent that."""
firefm = os.path.join(settings.ROOT,
'src/olympia/files/fixtures/files/firefm.xpi')
rdf = parse_xpi(open(firefm))
assert rdf['name'] == 'Fire.fm'
def test_parse_xpi():
"""Fire.fm can sometimes give us errors. Let's prevent that."""
firefm = os.path.join(settings.ROOT,
'src/olympia/files/fixtures/files/firefm.xpi')
rdf = parse_xpi(open(firefm))
assert rdf['name'] == 'Fire.fm'
