def test_mozilla_trademark_for_prefix_allowed(self, resolve_message): resolve_message.return_value = 'Notify for Mozilla' addon = amo.tests.addon_factory() file_obj = addon.current_version.file fixture = 'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi' with amo.tests.copy_file(fixture, file_obj.file_path): utils.parse_xpi(file_obj.file_path)
def test_parse_xpi_no_manifest(self): fake_zip = utils.make_xpi({'dummy': 'dummy'}) with mock.patch( 'olympia.files.utils.get_file' ) as get_file_mock, self.assertRaises(utils.NoManifestFound) as exc: get_file_mock.return_value = fake_zip utils.parse_xpi(None) assert isinstance(exc.exception, forms.ValidationError) assert exc.exception.message == ('No manifest.json found')
def test_mozilla_trademark_for_prefix_allowed(self, resolve_message): resolve_message.return_value = 'Notify for Mozilla' addon = amo.tests.addon_factory() file_obj = addon.current_version.all_files[0] fixture = ( 'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi') with amo.tests.copy_file(fixture, file_obj.file_path): utils.parse_xpi(file_obj.file_path)
def test_mozilla_trademark_for_prefix_not_allowed(self, resolve_message): resolve_message.return_value = 'Notify for Mozilla' addon = amo.tests.addon_factory() file_obj = addon.current_version.all_files[0] fixture = ( 'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi') with amo.tests.copy_file(fixture, file_obj.file_path): with self.assertRaises(forms.ValidationError): utils.parse_xpi(file_obj.file_path)
def test_mozilla_trademark_disallowed(self, resolve_message): resolve_message.return_value = 'Notify Mozilla' addon = amo.tests.addon_factory() file_obj = addon.current_version.file fixture = 'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi' with amo.tests.copy_file(fixture, file_obj.file_path): with pytest.raises(forms.ValidationError) as exc: utils.parse_xpi(file_obj.file_path) assert dict(exc.value.messages)['en-us'].startswith( 'Add-on names cannot contain the Mozilla or')
def test_mozilla_trademark_disallowed(self, resolve_message): resolve_message.return_value = 'Notify Mozilla' addon = amo.tests.addon_factory() file_obj = addon.current_version.all_files[0] fixture = ( 'src/olympia/files/fixtures/files/notify-link-clicks-i18n.xpi') with amo.tests.copy_file(fixture, file_obj.file_path): with pytest.raises(forms.ValidationError) as exc: utils.parse_xpi(file_obj.file_path) assert dict(exc.value.messages)['en-us'].startswith( u'Add-on names cannot contain the Mozilla or' )
def extract_optional_permissions(ids, **kw): log.info( '[%s@%s] Extracting permissions from Files, from id: %s to id: %s...' % (len(ids), extract_optional_permissions.rate_limit, ids[0], ids[-1])) files = File.objects.filter(pk__in=ids).no_transforms() # A user needs to be passed down to parse_xpi(), so we use the task user. user = UserProfile.objects.get(pk=settings.TASK_USER_ID) for file_ in files: try: log.info('Parsing File.id: %s @ %s' % (file_.pk, file_.current_file_path)) parsed_data = parse_xpi(file_.current_file_path, addon=file_.addon, user=user) optional_permissions = parsed_data.get('optional_permissions', []) if optional_permissions: log.info('Found %s optional permissions for: %s' % (len(optional_permissions), file_.pk)) WebextPermission.objects.update_or_create( defaults={'optional_permissions': optional_permissions}, file=file_) except Exception as err: log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def test_bump_version_in_manifest_json(file_obj): with amo.tests.copy_file( 'src/olympia/files/fixtures/files/webextension.xpi', file_obj.file_path): utils.update_version_number(file_obj, '0.0.1.1-signed') parsed = utils.parse_xpi(file_obj.file_path) assert parsed['version'] == '0.0.1.1-signed'
def test_bump_version_in_package_json(file_obj): with amo.tests.copy_file( 'src/olympia/files/fixtures/files/new-format-0.0.1.xpi', file_obj.file_path): utils.update_version_number(file_obj, '0.0.1.1-signed') parsed = utils.parse_xpi(file_obj.file_path) assert parsed['version'] == '0.0.1.1-signed'
def extract_webext_permissions(ids, **kw): log.info( '[%s@%s] Extracting permissions from Files, starting at id: %s...' % (len(ids), extract_webext_permissions.rate_limit, ids[0])) files = File.objects.filter(pk__in=ids).no_transforms() # A user needs to be passed down to parse_xpi(), so we use the task user. user = UserProfile.objects.get(pk=settings.TASK_USER_ID) for file_ in files: try: log.info('Parsing File.id: %s @ %s' % (file_.pk, file_.current_file_path)) parsed_data = parse_xpi(file_.current_file_path, user=user) permissions = parsed_data.get('permissions', []) # Add content_scripts host matches too. for script in parsed_data.get('content_scripts', []): permissions.extend(script.get('matches', [])) if permissions: log.info('Found %s permissions for: %s' % (len(permissions), file_.pk)) WebextPermission.objects.update_or_create( defaults={'permissions': permissions}, file=file_) except Exception as err: log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def extract_theme_properties(addon, channel): version = addon.find_latest_version(channel) if not version or not version.all_files: return {} try: parsed_data = parse_xpi( version.all_files[0].file_path, addon=addon, user=core.get_user()) except ValidationError: # If we can't parse the existing manifest safely return. return {} theme_props = parsed_data.get('theme', {}) # pre-process colors to convert chrome style colors and strip spaces theme_props['colors'] = dict( process_color_value(prop, color) for prop, color in theme_props.get('colors', {}).items()) # replace headerURL with path to existing background if 'images' in theme_props: if 'theme_frame' in theme_props['images']: header_url = theme_props['images'].pop('theme_frame') if 'headerURL' in theme_props['images']: header_url = theme_props['images'].pop('headerURL') if header_url: theme_props['images']['headerURL'] = '/'.join(( user_media_url('addons'), text_type(addon.id), text_type(version.id), header_url)) return theme_props
def test_bump_version_in_manifest_json(file_obj): create_switch('webextensions') with amo.tests.copy_file( 'src/olympia/files/fixtures/files/webextension.xpi', file_obj.file_path): utils.update_version_number(file_obj, '0.0.1.1-signed') parsed = utils.parse_xpi(file_obj.file_path) assert parsed['version'] == '0.0.1.1-signed'
def test_moz_signed_extension_no_strict_compat(self): addon = amo.tests.addon_factory() file_obj = addon.current_version.all_files[0] file_obj.update(is_mozilla_signed_extension=True) fixture = ('src/olympia/files/fixtures/files/' 'legacy-addon-already-signed-0.1.0.xpi') with amo.tests.copy_file(fixture, file_obj.file_path): parsed = utils.parse_xpi(file_obj.file_path) assert parsed['is_mozilla_signed_extension'] assert not parsed['strict_compatibility']
def test_moz_signed_extension_no_strict_compat(self): addon = amo.tests.addon_factory() file_obj = addon.current_version.all_files[0] file_obj.update(is_mozilla_signed_extension=True) fixture = ( 'src/olympia/files/fixtures/files/' 'legacy-addon-already-signed-0.1.0.xpi') with amo.tests.copy_file(fixture, file_obj.file_path): parsed = utils.parse_xpi(file_obj.file_path) assert parsed['is_mozilla_signed_extension'] assert not parsed['strict_compatibility']
def test_moz_signed_extension_reuse_strict_compat(self): addon = amo.tests.addon_factory() user = amo.tests.user_factory(email='*****@*****.**') file_obj = addon.current_version.all_files[0] file_obj.update(is_mozilla_signed_extension=True) fixture = ('src/olympia/files/fixtures/files/' 'legacy-addon-already-signed-strict-compat-0.1.0.xpi') with amo.tests.copy_file(fixture, file_obj.file_path): parsed = utils.parse_xpi(file_obj.file_path, user=user) assert parsed['is_mozilla_signed_extension'] # We set `strictCompatibility` in install.rdf assert parsed['strict_compatibility']
def test_moz_signed_extension_reuse_strict_compat(self): addon = amo.tests.addon_factory() user = amo.tests.user_factory(email='*****@*****.**') file_obj = addon.current_version.all_files[0] file_obj.update(is_mozilla_signed_extension=True) fixture = ( 'src/olympia/files/fixtures/files/' 'legacy-addon-already-signed-strict-compat-0.1.0.xpi') with amo.tests.copy_file(fixture, file_obj.file_path): parsed = utils.parse_xpi(file_obj.file_path, user=user) assert parsed['is_mozilla_signed_extension'] # We set `strictCompatibility` in install.rdf assert parsed['strict_compatibility']
def test_bump_version_in_manifest_json(file_obj): AppVersion.objects.create(application=amo.FIREFOX.id, version=amo.DEFAULT_WEBEXT_MIN_VERSION) AppVersion.objects.create(application=amo.FIREFOX.id, version=amo.DEFAULT_WEBEXT_MAX_VERSION) AppVersion.objects.create(application=amo.ANDROID.id, version=amo.DEFAULT_WEBEXT_MIN_VERSION_ANDROID) AppVersion.objects.create(application=amo.ANDROID.id, version=amo.DEFAULT_WEBEXT_MAX_VERSION) with amo.tests.copy_file( 'src/olympia/files/fixtures/files/webextension.xpi', file_obj.file_path): utils.update_version_number(file_obj, '0.0.1.1-signed') parsed = utils.parse_xpi(file_obj.file_path) assert parsed['version'] == '0.0.1.1-signed'
def extract_theme_properties(addon, channel): version = addon.find_latest_version(channel) if not version or not version.all_files: return {} try: parsed_data = parse_xpi( version.all_files[0].file_path, addon=addon, user=core.get_user()) except ValidationError: # If we can't parse the existing manifest safely return. return {} theme_props = parsed_data.get('theme', {}) # pre-process colors to convert chrome style colors and strip spaces theme_props['colors'] = dict( process_color_value(prop, color) for prop, color in theme_props.get('colors', {}).items()) return theme_props
def extract_webext_permissions(ids, **kw): log.info('[%s@%s] Extracting permissions from Files, starting at id: %s...' % (len(ids), extract_webext_permissions.rate_limit, ids[0])) files = File.objects.filter(pk__in=ids).no_transforms() for file_ in files: try: log.info('Parsing File.id: %s @ %s' % (file_.pk, file_.current_file_path)) parsed_data = parse_xpi(file_.current_file_path, check=False) permissions = parsed_data.get('permissions') if permissions: log.info('Found %s permissions for: %s' % (len(permissions), file_.pk)) WebextPermission.objects.update_or_create( defaults={'permissions': permissions}, file=file_) except Exception, err: log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def extract_webext_permissions(ids, **kw): log.info( '[%s@%s] Extracting permissions from Files, starting at id: %s...' % (len(ids), extract_webext_permissions.rate_limit, ids[0])) files = File.objects.filter(pk__in=ids).no_transforms() for file_ in files: try: log.info('Parsing File.id: %s @ %s' % (file_.pk, file_.current_file_path)) parsed_data = parse_xpi(file_.current_file_path, check=False) permissions = parsed_data.get('permissions') if permissions: log.info('Found %s permissions for: %s' % (len(permissions), file_.pk)) WebextPermission.objects.update_or_create( defaults={'permissions': permissions}, file=file_) except Exception, err: log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def extract_theme_properties(addon, channel): version = addon.find_latest_version(channel) if not version or not version.all_files: return {} try: parsed_data = parse_xpi( version.all_files[0].file_path, addon=addon, user=core.get_user()) except ValidationError: # If we can't parse the existing manifest safely return. return {} theme_props = parsed_data.get('theme', {}) # pre-process colors to deprecated colors; strip spaces. theme_props['colors'] = dict( process_color_value(prop, color) for prop, color in theme_props.get('colors', {}).items()) # upgrade manifest from deprecated headerURL to theme_frame if 'headerURL' in theme_props.get('images', {}): url = theme_props['images'].pop('headerURL') theme_props['images']['theme_frame'] = url return theme_props
def populate_e10s_feature_compatibility(ids, **kwargs): log.info( '[%s@%s] Populating e10s feature compatibility ' 'on addons starting w/ id: %s...' % (len(ids), populate_e10s_feature_compatibility.rate_limit, ids[0])) addons = Addon.unfiltered.filter(pk__in=ids) for addon in addons: file_ = addon.get_latest_file() if file_: try: parsed = parse_xpi(file_.file_path, addon=addon) except ValidationError: log.warn('Could not parse XPI for addon %s, file %s' % (addon.pk, file_.pk)) continue feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create( addon=addon)[0]) feature_compatibility.update(e10s=parsed['e10s_compatibility'])
def populate_e10s_feature_compatibility(ids, **kwargs): log.info( '[%s@%s] Populating e10s feature compatibility ' 'on addons starting w/ id: %s...' % ( len(ids), populate_e10s_feature_compatibility.rate_limit, ids[0])) addons = Addon.unfiltered.filter(pk__in=ids) for addon in addons: file_ = addon.get_latest_file() if file_: try: parsed = parse_xpi(file_.file_path, addon=addon) except ValidationError: log.warn('Could not parse XPI for addon %s, file %s' % ( addon.pk, file_.pk)) continue feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0] ) feature_compatibility.update(e10s=parsed['e10s_compatibility'])
def extract_webext_permissions(ids, **kw): log.info('[%s@%s] Extracting permissions from Files, starting at id: %s...' % (len(ids), extract_webext_permissions.rate_limit, ids[0])) files = File.objects.filter(pk__in=ids).no_transforms() # A user needs to be passed down to parse_xpi(), so we use the task user. user = UserProfile.objects.get(pk=settings.TASK_USER_ID) for file_ in files: try: log.info('Parsing File.id: %s @ %s' % (file_.pk, file_.current_file_path)) parsed_data = parse_xpi(file_.current_file_path, user=user) permissions = parsed_data.get('permissions', []) # Add content_scripts host matches too. for script in parsed_data.get('content_scripts', []): permissions.extend(script.get('matches', [])) if permissions: log.info('Found %s permissions for: %s' % (len(permissions), file_.pk)) WebextPermission.objects.update_or_create( defaults={'permissions': permissions}, file=file_) except Exception as err: log.error('Failed to extract: %s, error: %s' % (file_.pk, err))
def test_bump_version_in_alt_install_rdf(file_obj): with amo.tests.copy_file('src/olympia/files/fixtures/files/alt-rdf.xpi', file_obj.file_path): utils.update_version_number(file_obj, '2.1.106.1-signed') parsed = utils.parse_xpi(file_obj.file_path) assert parsed['version'] == '2.1.106.1-signed'
def test_bump_version_in_install_rdf(file_obj): with amo.tests.copy_file("src/olympia/files/fixtures/files/jetpack.xpi", file_obj.file_path): utils.update_version_number(file_obj, "1.3.1-signed") parsed = utils.parse_xpi(file_obj.file_path) assert parsed["version"] == "1.3.1-signed"
def test_bump_version_in_manifest_json(file_obj): with amo.tests.copy_file("src/olympia/files/fixtures/files/webextension.xpi", file_obj.file_path): utils.update_version_number(file_obj, "0.0.1.1-signed") parsed = utils.parse_xpi(file_obj.file_path) assert parsed["version"] == "0.0.1.1-signed"
def repack_fileupload(results, upload_pk): log.info('Starting task to repackage FileUpload %s', upload_pk) upload = FileUpload.objects.get(pk=upload_pk) # When a FileUpload is created and a file added to it, if it's a xpi/zip, # it should be move to upload.path, and it should have a .zip extension, # so we only need to care about that extension here. # We don't trust upload.name: it's the original filename as used by the # developer, so it could be something else. if upload.path.endswith('.zip'): timer = StopWatch('files.tasks.repack_fileupload.') timer.start() # tempdir must *not* be on TMP_PATH, we want local fs instead. It will be # deleted automatically once we exit the context manager. with tempfile.TemporaryDirectory( prefix='repack_fileupload_extract') as tempdir: try: extract_zip(upload.path, tempdir=tempdir) if waffle.switch_is_active('enable-manifest-normalization'): manifest = Path(tempdir) / 'manifest.json' if manifest.exists(): try: xpi_data = parse_xpi(upload.path, minimal=True) if not xpi_data.get('is_mozilla_signed_extension', False): json_data = ManifestJSONExtractor( manifest.read_bytes()).data manifest.write_text( json.dumps(json_data, indent=2)) except Exception: # If we cannot normalize the manifest file, we skip # this step and let the linter catch the exact # cause in order to return a more appropriate error # than "unexpected error", which would happen if # this task was handling the error itself. pass except Exception as exc: # Something bad happened, maybe we couldn't parse the zip file. # @validation_task should ensure the exception is caught and # transformed in a generic error message for the developer, so we # just log it and re-raise. log.exception('Could not extract upload %s for repack.', upload_pk, exc_info=exc) raise timer.log_interval('1.extracted') log.info('Zip from upload %s extracted, repackaging', upload_pk) # We'll move the file to its final location below with move_stored_file(), # so don't let tempfile delete it. file_ = tempfile.NamedTemporaryFile(dir=settings.TMP_PATH, suffix='.zip', delete=False) shutil.make_archive( os.path.splitext(file_.name)[0], 'zip', tempdir) with open(file_.name, 'rb') as f: upload.hash = 'sha256:%s' % get_sha256(f) timer.log_interval('2.repackaged') log.info('Zip from upload %s repackaged, moving file back', upload_pk) storage.move_stored_file(file_.name, upload.path) timer.log_interval('3.moved') upload.save() timer.log_interval('4.end') else: log.info('Not repackaging upload %s, it is not a zip file.', upload_pk) return results
def test_parse_xpi(): """Fire.fm can sometimes give us errors. Let's prevent that.""" firefm = os.path.join(settings.ROOT, 'src/olympia/files/fixtures/files/firefm.xpi') rdf = parse_xpi(open(firefm)) assert rdf['name'] == 'Fire.fm'