def test_ip_log(self):
addon = Addon.objects.get()
assert IPLog.objects.count() == 0
# Creating an activity log for an action without store_ip=True doesn't
# create an IPLog.
action = amo.LOG.REJECT_VERSION
assert not getattr(action, 'store_ip', False)
with core.override_remote_addr('127.0.4.8'):
activity = ActivityLog.create(
action,
addon,
addon.current_version,
user=self.request.user,
)
assert IPLog.objects.count() == 0
# Creating an activity log for an action *with* store_ip=True *does*
# create an IPLog.
action = amo.LOG.ADD_VERSION
assert getattr(action, 'store_ip', False)
with core.override_remote_addr('15.16.23.42'):
activity = ActivityLog.create(
action,
addon,
addon.current_version,
user=self.request.user,
)
assert IPLog.objects.count() == 1
ip_log = IPLog.objects.get()
assert ip_log.activity_log == activity
assert ip_log.ip_address == '15.16.23.42'
def test_override_remote_addr():
original = core.get_remote_addr()
with core.override_remote_addr('some other value'):
assert core.get_remote_addr() == 'some other value'
assert core.get_remote_addr() == original
def test_to_string_num_queries_model_depending_on_addon(self):
addon = Addon.objects.get()
addon2 = addon_factory()
with core.override_remote_addr('1.1.1.1'):
ActivityLog.create(
amo.LOG.ADD_VERSION,
addon,
addon.current_version,
user=self.request.user,
)
ActivityLog.create(
amo.LOG.ADD_VERSION,
addon2,
addon2.current_version,
user=user_factory(),
)
with self.assertNumQueries(6):
# - 1 for all activities
# - 1 for all users
# - 1 for all addons
# - 1 for all add-on translations
# - 1 for all versions
# - 1 for all versions translations
activities = ActivityLog.objects.for_addons([addon, addon2
]).order_by('pk')
assert len(activities) == 2
addon_url = 'http://testserver/en-US/firefox/addon/a3615/'
assert activities[0].to_string() == (
f'<a href="{addon_url}versions/">Version 2.1.072</a> added to '
f'<a href="{addon_url}">Delicious Bookmarks</a>.')
assert activities[1].to_string()
def log_updates(self, num, version_string='1'):
version = Version.objects.create(version=version_string,
addon=self.addon)
for i in range(num):
with core.override_remote_addr('127.0.0.1'):
ActivityLog.create(amo.LOG.ADD_VERSION, self.addon, version)
def from_upload(cls,
upload,
addon,
selected_apps,
channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.addons.models import AddonReviewerFlags
from olympia.addons.utils import RestrictionChecker
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
if upload.addon and upload.addon != addon:
raise VersionCreateError(
'FileUpload was made for a different Addon')
if not upload.user or not upload.ip_address or not upload.source:
raise VersionCreateError(
'FileUpload does not have some required fields')
if not upload.user.last_login_ip or not upload.user.email:
raise VersionCreateError(
'FileUpload user does not have some required fields')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (
'This version has been signed with Mozilla internal certificate.'
)
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
# The following log statement is used by foxsec-pipeline.
# We override the IP because it might be called from a task and we
# want the original IP from the submitter.
log.info(
f'New version: {version!r} ({version.id}) from {upload!r}',
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
},
)
activity.log_create(amo.LOG.ADD_VERSION,
version,
addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# Pre-generate _compatible_apps property to avoid accidentally
# triggering queries with that instance later.
version._compatible_apps = compatible_apps
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [
File.from_upload(
upload=upload,
version=version,
parsed_data=parsed_data,
)
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to its permanent location, delete it
# from storage. Keep the FileUpload instance (it gets cleaned up by a
# cron eventually some time after its creation, in amo.cron.gc()),
# making sure it's associated with the add-on instance.
storage.delete(upload.path)
upload.path = ''
if upload.addon is None:
upload.addon = addon
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (waffle.switch_is_active('enable-yara')
or waffle.switch_is_active('enable-customs')
or waffle.switch_is_active('enable-wat')):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version))
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Reset add-on reviewer flags to disable auto-approval and require
# admin code review if the package has already been signed by mozilla.
reviewer_flags_defaults = {}
is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension')
if upload.validation_timeout:
reviewer_flags_defaults['needs_admin_code_review'] = True
if is_mozilla_signed and addon.type != amo.ADDON_LPAPP:
reviewer_flags_defaults['needs_admin_code_review'] = True
reviewer_flags_defaults['auto_approval_disabled'] = True
# Check if the approval should be restricted
if not RestrictionChecker(upload=upload).is_auto_approval_allowed():
flag = ('auto_approval_disabled'
if channel == amo.RELEASE_CHANNEL_LISTED else
'auto_approval_disabled_unlisted')
reviewer_flags_defaults[flag] = True
if reviewer_flags_defaults:
AddonReviewerFlags.objects.update_or_create(
addon=addon, defaults=reviewer_flags_defaults)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def from_upload(cls, upload, addon, selected_apps, channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(
channel=channel, exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (u'This version has been signed with '
u'Mozilla internal certificate.')
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
log.info(
'New version: %r (%s) from %r' % (version, version.id, upload),
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
}
)
activity.log_create(
amo.LOG.ADD_VERSION, version, addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
# For backwards compatibility. We removed specific platform
# support during submission but we don't handle it any different
# beyond that yet. That means, we're going to simply set it
# to `PLATFORM_ALL` and also have the backend create separate
# files for each platform. Cleaning that up is another step.
# Given the timing on this, we don't care about updates to legacy
# add-ons as well.
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [File.from_upload(
upload=upload, version=version, platform=amo.PLATFORM_ALL.id,
parsed_data=parsed_data
)]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
upload.path = ''
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (
waffle.switch_is_active('enable-yara') or
waffle.switch_is_active('enable-customs') or
waffle.switch_is_active('enable-wat')
):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version)
)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME and
channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=version,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def test_known_ip_adresses(self):
self.user.update(last_login_ip='127.1.2.3')
Rating.objects.create(addon=addon_factory(),
user=self.user,
ip_address='127.1.2.3')
dummy_addon = addon_factory()
Rating.objects.create(
addon=dummy_addon,
version=dummy_addon.current_version,
user=self.user,
ip_address='128.1.2.3',
)
Rating.objects.create(
addon=dummy_addon,
version=version_factory(addon=dummy_addon),
user=self.user,
ip_address='129.1.2.4',
)
Rating.objects.create(addon=addon_factory(),
user=self.user,
ip_address='130.1.2.4')
Rating.objects.create(addon=addon_factory(),
user=self.user,
ip_address='130.1.2.4')
Rating.objects.create(addon=dummy_addon,
user=user_factory(),
ip_address='255.255.0.0')
with core.override_remote_addr('15.16.23.42'):
ActivityLog.create(amo.LOG.ADD_VERSION,
dummy_addon,
user=self.user)
UserRestrictionHistory.objects.create(user=self.user,
last_login_ip='4.8.15.16')
UserRestrictionHistory.objects.create(user=self.user,
ip_address='172.0.0.2')
model_admin = UserAdmin(UserProfile, admin.site)
doc = pq(model_admin.known_ip_adresses(self.user))
result = doc('ul li').text().split()
assert len(result) == 7
assert set(result) == {
'130.1.2.4',
'128.1.2.3',
'129.1.2.4',
'127.1.2.3',
'15.16.23.42',
'172.0.0.2',
'4.8.15.16',
}
# Duplicates are ignored
Rating.objects.create(
addon=dummy_addon,
version=version_factory(addon=dummy_addon),
user=self.user,
ip_address='127.1.2.3',
)
with core.override_remote_addr('172.0.0.2'):
ActivityLog.create(amo.LOG.ADD_VERSION,
dummy_addon,
user=self.user)
UserRestrictionHistory.objects.create(user=self.user,
last_login_ip='15.16.23.42')
UserRestrictionHistory.objects.create(user=self.user,
ip_address='4.8.15.16')
doc = pq(model_admin.known_ip_adresses(self.user))
result = doc('ul li').text().split()
assert len(result) == 7
assert set(result) == {
'130.1.2.4',
'128.1.2.3',
'129.1.2.4',
'127.1.2.3',
'15.16.23.42',
'172.0.0.2',
'4.8.15.16',
}
def create_version(self, addon=None):
from olympia.addons.models import Addon
from olympia.files.models import FileUpload
from olympia.files.utils import parse_addon
from olympia.versions.models import Version
from olympia.versions.utils import get_next_version_number
version_number = '1.0'
# If passing an existing add-on, we need to bump the version number
# to avoid clashes, and also perform a few checks.
if addon is not None:
# Obviously we want an add-on with the right type.
if addon.type != amo.ADDON_SITE_PERMISSION:
raise ImproperlyConfigured(
'SitePermissionVersionCreator was instantiated with non '
'site-permission add-on'
)
# If the user isn't an author, something is wrong.
if not addon.authors.filter(pk=self.user.pk).exists():
raise ImproperlyConfigured(
'SitePermissionVersionCreator was instantiated with a '
'bogus addon/user'
)
# Changing the origins isn't supported at the moment.
latest_version = addon.find_latest_version(
exclude=(), channel=amo.RELEASE_CHANNEL_UNLISTED
)
previous_origins = sorted(
latest_version.installorigin_set.all().values_list('origin', flat=True)
)
if previous_origins != self.install_origins:
raise ImproperlyConfigured(
'SitePermissionVersionCreator was instantiated with an '
'addon that has different origins'
)
version_number = get_next_version_number(addon)
# Create the manifest, with more user-friendly name & description built
# from install_origins/site_permissions, and then the zipfile with that
# manifest inside.
manifest_data = self._create_manifest(version_number)
file_obj = self._create_zipfile(manifest_data)
# Parse the zip we just created. The user needs to be the Mozilla User
# because regular submissions of this type of add-on is forbidden to
# normal users.
parsed_data = parse_addon(
file_obj,
addon=addon,
user=get_task_user(),
)
with core.override_remote_addr(self.remote_addr):
if addon is None:
# Create the Addon instance (without a Version/File at this point).
addon = Addon.initialize_addon_from_upload(
data=parsed_data,
upload=file_obj,
channel=amo.RELEASE_CHANNEL_UNLISTED,
user=self.user,
)
# Create the FileUpload that will become the File+Version.
upload = FileUpload.from_post(
file_obj,
filename=file_obj.name,
size=file_obj.size,
addon=addon,
version=version_number,
channel=amo.RELEASE_CHANNEL_UNLISTED,
user=self.user,
source=amo.UPLOAD_SOURCE_GENERATED,
)
# And finally create the Version instance from the FileUpload.
return Version.from_upload(
upload,
addon,
amo.RELEASE_CHANNEL_UNLISTED,
selected_apps=[x[0] for x in amo.APPS_CHOICES],
parsed_data=parsed_data,
)
