class FileValidation(ModelBase):
id = PositiveAutoField(primary_key=True)
file = models.OneToOneField(File,
related_name='validation',
on_delete=models.CASCADE)
valid = models.BooleanField(default=False)
errors = models.IntegerField(default=0)
warnings = models.IntegerField(default=0)
notices = models.IntegerField(default=0)
validation = models.TextField()
class Meta:
db_table = 'file_validation'
@classmethod
def from_json(cls, file, validation):
if isinstance(validation, str):
validation = json.loads(validation)
if 'metadata' in validation:
if (validation['metadata'].get('contains_binary_extension')
or validation['metadata'].get('contains_binary_content')):
file.update(binary=True)
if validation['metadata'].get('binary_components'):
file.update(binary_components=True)
# Delete any past results.
# We most often wind up with duplicate results when multiple requests
# for the same validation data are POSTed at the same time, which we
# currently do not have the ability to track.
cls.objects.filter(file=file).delete()
return cls.objects.create(file=file,
validation=json.dumps(validation),
errors=validation['errors'],
warnings=validation['warnings'],
notices=validation['notices'],
valid=validation['errors'] == 0)
@property
def processed_validation(self):
"""Return processed validation results as expected by the frontend."""
# Import loop.
from olympia.devhub.utils import process_validation
return process_validation(json.loads(self.validation),
file_hash=self.file.original_hash,
channel=self.file.version.channel)
class EmailUserRestriction(
GetErrorMessageMixin, NormalizeEmailMixin, ModelBase):
id = PositiveAutoField(primary_key=True)
email_pattern = models.CharField(
_('Email Pattern'),
max_length=100,
help_text=_(
'Either enter full domain or email that should be blocked or use '
' glob-style wildcards to match other patterns.'
' E.g "@*.mail.com"\n'
' Please note that we do not include "@" in the match so you '
' should do that in the pattern.'))
error_message = _('The email address used for your account is not '
'allowed for add-on submission.')
class Meta:
db_table = 'users_user_email_restriction'
def __str__(self):
return str(self.email_pattern)
@classmethod
def allow_request(cls, request):
"""
Return whether the specified request should be allowed to submit
add-ons.
"""
if not request.user.is_authenticated:
return False
return cls.allow_email(cls.normalize_email(request.user.email))
@classmethod
def allow_email(cls, email):
"""
Return whether the specified email should be allowed to submit add-ons.
"""
restrictions = EmailUserRestriction.objects.all()
for restriction in restrictions:
if fnmatchcase(email, restriction.email_pattern):
log.info('Restricting request from %s %s (%s)',
'email', email,
'email_pattern=%s' % restriction.email_pattern)
return False
return True
class UpdateCount(StatsSearchMixin, models.Model):
id = PositiveAutoField(primary_key=True)
# Has an index `addon_id` in our dev, stage and prod database
addon = models.ForeignKey('addons.Addon')
# Has an index named `count` in our dev, stage and prod database
count = models.PositiveIntegerField(db_index=True)
# Has an index named `date` in our dev, stage and prod database
date = models.DateField(db_index=True)
versions = JSONField(db_column='version', null=True)
statuses = JSONField(db_column='status', null=True)
applications = JSONField(db_column='application', null=True)
oses = JSONField(db_column='os', null=True)
locales = JSONField(db_column='locale', null=True)
class Meta:
db_table = 'update_counts'
class AddonTag(ModelBase):
id = PositiveAutoField(primary_key=True)
addon = models.ForeignKey(
'addons.Addon', related_name='addon_tags', on_delete=models.CASCADE
)
tag = models.ForeignKey(Tag, related_name='addon_tags', on_delete=models.CASCADE)
class Meta:
db_table = 'users_tags_addons'
indexes = [
models.Index(fields=('tag',), name='tag_id'),
models.Index(fields=('addon',), name='addon_id'),
]
constraints = [
models.UniqueConstraint(fields=('tag', 'addon'), name='tag_id_2'),
]
class CollectionAddon(ModelBase):
id = PositiveAutoField(primary_key=True)
addon = models.ForeignKey(Addon, on_delete=models.CASCADE)
collection = models.ForeignKey(Collection, on_delete=models.CASCADE)
# category (deprecated: for "Fashion Your Firefox")
comments = LinkifiedField(null=True)
user = models.ForeignKey(UserProfile, null=True, on_delete=models.CASCADE)
ordering = models.PositiveIntegerField(
default=0,
help_text='Add-ons are displayed in ascending order '
'based on this field.')
class Meta(ModelBase.Meta):
db_table = 'addons_collections'
indexes = [
models.Index(fields=('collection', 'created'), name='created_idx'),
models.Index(fields=('addon', ), name='addon_id'),
models.Index(fields=('collection', ), name='collection_id'),
models.Index(fields=('user', ), name='user_id'),
]
constraints = [
models.UniqueConstraint(fields=('addon', 'collection'),
name='addon_id_2'),
]
@staticmethod
def post_save(sender, instance, **kwargs):
"""Update Collection.addon_count and reindex add-on if the collection
is featured."""
if kwargs.get('raw'):
return
if instance.collection.listed:
activity.log_create(amo.LOG.ADD_TO_COLLECTION, instance.addon,
instance.collection)
Collection.post_save(sender, instance.collection, **kwargs)
@staticmethod
def post_delete(sender, instance, **kwargs):
from . import tasks
if kwargs.get('raw'):
return
if instance.collection.listed:
activity.log_create(amo.LOG.REMOVE_FROM_COLLECTION, instance.addon,
instance.collection)
tasks.collection_meta.delay(instance.collection.id)
class RereviewQueueTheme(ModelBase):
id = PositiveAutoField(primary_key=True)
theme = models.ForeignKey(Persona, on_delete=models.CASCADE)
header = models.CharField(max_length=72, blank=True, default='')
# Holds whether this reuploaded theme is a duplicate.
dupe_persona = models.ForeignKey(Persona,
null=True,
related_name='dupepersona',
on_delete=models.CASCADE)
# The order of those managers is very important: please read the lengthy
# comment above the Addon managers declaration/instantiation.
unfiltered = RereviewQueueThemeManager(include_deleted=True)
objects = RereviewQueueThemeManager()
class Meta:
db_table = 'rereview_queue_theme'
# This is very important: please read the lengthy comment in Addon.Meta
# description
base_manager_name = 'unfiltered'
def __str__(self):
return str(self.id)
@property
def header_path(self):
"""Return the path to the header image."""
return self.theme._image_path(self.header or self.theme.header)
@property
def footer_path(self):
"""Return the path to the optional footer image."""
footer = self.footer or self.theme.footer
return footer and self.theme._image_path(footer) or ''
@property
def header_url(self):
"""Return the url of the header imager."""
return self.theme._image_url(self.header or self.theme.header)
@property
def footer_url(self):
"""Return the url of the optional footer image."""
footer = self.footer or self.theme.footer
return footer and self.theme._image_url(footer) or ''
class RssKey(models.Model):
id = PositiveAutoField(primary_key=True)
key = models.UUIDField(db_column='rsskey',
unique=True,
null=True,
default=uuid.uuid4)
addon = models.ForeignKey(Addon,
null=True,
unique=True,
on_delete=models.CASCADE)
user = models.ForeignKey(UserProfile,
null=True,
unique=True,
on_delete=models.CASCADE)
created = models.DateField(default=datetime.now)
class Meta:
db_table = 'hubrsskeys'
class FeaturedCollection(ModelBase):
id = PositiveAutoField(primary_key=True)
application = models.PositiveIntegerField(choices=amo.APPS_CHOICES,
db_column='application_id')
collection = models.ForeignKey(Collection)
locale = models.CharField(max_length=10, null=True)
class Meta:
db_table = 'featured_collections'
def __str__(self):
return u'%s (%s: %s)' % (self.collection, self.application,
self.locale)
@staticmethod
def post_save_or_delete(sender, instance, **kwargs):
Collection.update_featured_status(FeaturedCollection,
instance.collection, **kwargs)
class DraftComment(ModelBase):
"""A model that allows us to draft comments for reviews before we have
an ActivityLog instance ready.
This is being used by the commenting API by the code-manager.
"""
id = PositiveAutoField(primary_key=True)
version = models.ForeignKey(Version, on_delete=models.CASCADE)
user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
filename = models.CharField(max_length=255, null=True, blank=True)
lineno = models.PositiveIntegerField(null=True)
canned_response = models.ForeignKey(
CannedResponse, null=True, default=None,
on_delete=models.SET_DEFAULT)
comment = models.TextField(blank=True)
class Meta:
db_table = 'log_activity_comment_draft'
class GroupUser(models.Model):
id = PositiveAutoField(primary_key=True)
group = models.ForeignKey(Group, on_delete=models.CASCADE)
user = models.ForeignKey('users.UserProfile', on_delete=models.CASCADE)
class Meta:
db_table = u'groups_users'
def invalidate_groups_list(self):
"""Callback to invalidate user.groups_list when creating/deleting GroupUser
instances for this user."""
try:
# groups_list is a @cached_property, delete it to force it to be
# refreshed (ignore AttributeError, that just means it has not been
# accessed yet).
del self.user.groups_list
except AttributeError:
pass
class IPNetworkUserRestriction(ModelBase):
id = PositiveAutoField(primary_key=True)
network = CIDRField(
blank=True,
null=True,
help_text=_(
'Enter a valid IPv6 or IPv6 CIDR network range, eg. 127.0.0.1/28'))
error_message = _('Multiple add-ons violating our policies have been'
' submitted from your location. The IP address has been'
' blocked.')
class Meta:
db_table = 'users_user_network_restriction'
def __str__(self):
return str(self.network)
@classmethod
def allow_request(cls, request):
"""
Return whether the specified request should be allowed to submit
add-ons.
"""
try:
remote_addr = ipaddress.ip_address(request.META.get('REMOTE_ADDR'))
if request.user:
user_last_login_ip = ipaddress.ip_address(
request.user.last_login_ip)
except ValueError:
# If we don't have a valid ip address, let's deny
return False
restrictions = IPNetworkUserRestriction.objects.all()
for restriction in restrictions:
if (remote_addr in restriction.network
or user_last_login_ip in restriction.network):
log.info('Restricting request from %s %s, %s %s (%s)', 'ip',
remote_addr, 'last_login_ip', user_last_login_ip,
'network=%s' % restriction.network)
return False
return True
class CannedResponse(ModelBase):
id = PositiveAutoField(primary_key=True)
name = models.CharField(max_length=255)
response = models.TextField()
sort_group = models.CharField(max_length=255)
type = models.PositiveIntegerField(
choices=amo.CANNED_RESPONSE_TYPE_CHOICES.items(), db_index=True,
default=0)
# Category is used only by code-manager
category = models.PositiveIntegerField(
choices=amo.CANNED_RESPONSE_CATEGORY_CHOICES.items(),
default=amo.CANNED_RESPONSE_CATEGORY_OTHER)
class Meta:
db_table = 'cannedresponses'
def __str__(self):
return str(self.name)
class Tag(ModelBase):
id = PositiveAutoField(primary_key=True)
tag_text = models.CharField(max_length=128)
addons = models.ManyToManyField(
'addons.Addon', through='AddonTag', related_name='tags'
)
num_addons = models.IntegerField(default=0)
class Meta:
db_table = 'tags'
ordering = ('tag_text',)
constraints = [models.UniqueConstraint(fields=('tag_text',), name='tag_text')]
def __str__(self):
return self.tag_text
@property
def popularity(self):
return self.num_addons
def can_reverse(self):
try:
self.get_url_path()
return True
except NoReverseMatch:
return False
def get_url_path(self):
return reverse('tags.detail', args=[self.tag_text])
def add_tag(self, addon):
AddonTag.objects.get_or_create(addon=addon, tag=self)
activity.log_create(amo.LOG.ADD_TAG, self, addon)
def remove_tag(self, addon):
for addon_tag in AddonTag.objects.filter(addon=addon, tag=self):
addon_tag.delete()
activity.log_create(amo.LOG.REMOVE_TAG, self, addon)
def update_stat(self):
self.num_addons = self.addons.count()
self.save()
class DownloadCount(StatsSearchMixin, models.Model):
id = PositiveAutoField(primary_key=True)
# has an index `addon_id` on this column...
addon = models.ForeignKey('addons.Addon')
# has an index named `count` in dev, stage and prod
count = models.PositiveIntegerField(db_index=True)
date = models.DateField()
sources = JSONField(db_column='src', null=True)
class Meta:
db_table = 'download_counts'
# additional indices on this table (in dev, stage and prod):
# * KEY `addon_and_count` (`addon_id`,`count`)
# * KEY `addon_date_idx` (`addon_id`,`date`)
# in our (dev, stage and prod) database:
# UNIQUE KEY `date_2` (`date`,`addon_id`)
unique_together = ('date', 'addon')
class DownloadCount(StatsSearchMixin, models.Model):
id = PositiveAutoField(primary_key=True)
addon = models.ForeignKey('addons.Addon', on_delete=models.CASCADE)
count = models.PositiveIntegerField()
date = models.DateField()
sources = JSONField(db_column='src', null=True)
class Meta:
db_table = 'download_counts'
indexes = [
# FIXME: some of these might redundant. See #5712
models.Index(fields=('count', ), name='count'),
models.Index(fields=('addon', ), name='addon_id'),
models.Index(fields=('addon', 'count'), name='addon_and_count'),
models.Index(fields=('addon', 'date'), name='addon_date_idx')
]
constraints = [
models.UniqueConstraint(fields=['date', 'addon'], name='date_2'),
]
class DiscoveryModule(ModelBase):
"""
Keeps the application, ordering, and locale metadata for a module.
The modules are defined statically in modules.py and linked to a database
row through the module's name.
"""
id = PositiveAutoField(primary_key=True)
app = models.PositiveIntegerField(choices=amo.APPS_CHOICES,
db_column='app_id')
module = models.CharField(max_length=255)
ordering = models.IntegerField(null=True, blank=True)
locales = models.CharField(max_length=255, blank=True, default='')
class Meta:
db_table = 'discovery_modules'
unique_together = ('app', 'module')
def __unicode__(self):
return u'%s (%s)' % (self.module, self.get_app_display())
class ApplicationsVersions(models.Model):
id = PositiveAutoField(primary_key=True)
application = models.PositiveIntegerField(choices=amo.APPS_CHOICES,
db_column='application_id')
version = models.ForeignKey(
Version, related_name='apps', on_delete=models.CASCADE)
min = models.ForeignKey(
AppVersion, db_column='min', related_name='min_set',
on_delete=models.CASCADE)
max = models.ForeignKey(
AppVersion, db_column='max', related_name='max_set',
on_delete=models.CASCADE)
class Meta:
db_table = 'applications_versions'
constraints = [
models.UniqueConstraint(fields=('application', 'version'),
name='application_id'),
]
def get_application_display(self):
return str(amo.APPS_ALL[self.application].pretty)
def get_latest_application_version(self):
return (
AppVersion.objects
.filter(
~models.Q(version__contains='*'),
application=self.application)
.order_by('-version_int')
.first())
def __str__(self):
if (self.version.is_compatible_by_default and
self.version.is_compatible_app(amo.APP_IDS[self.application])):
return ugettext(u'{app} {min} and later').format(
app=self.get_application_display(),
min=self.min
)
return u'%s %s - %s' % (self.get_application_display(),
self.min, self.max)
class UpdateCount(StatsSearchMixin, models.Model):
id = PositiveAutoField(primary_key=True)
addon = models.ForeignKey('addons.Addon', on_delete=models.CASCADE)
count = models.PositiveIntegerField()
date = models.DateField()
versions = JSONField(db_column='version', null=True)
statuses = JSONField(db_column='status', null=True)
applications = JSONField(db_column='application', null=True)
oses = JSONField(db_column='os', null=True)
locales = JSONField(db_column='locale', null=True)
class Meta:
db_table = 'update_counts'
indexes = [
# FIXME: some of these might redundant. See #5712
models.Index(fields=('count', ), name='count'),
models.Index(fields=('addon', ), name='addon_id'),
models.Index(fields=('date', ), name='date'),
models.Index(fields=('addon', 'count'), name='addon_and_count'),
models.Index(fields=('addon', 'date'), name='addon_date_idx')
]
class ActivityLogToken(ModelBase):
id = PositiveAutoField(primary_key=True)
version = models.ForeignKey(Version, related_name='token', on_delete=models.CASCADE)
user = models.ForeignKey(
'users.UserProfile',
related_name='activity_log_tokens',
on_delete=models.CASCADE,
)
uuid = models.UUIDField(default=uuid.uuid4, unique=True)
use_count = models.IntegerField(
default=0, help_text='Stores the number of times the token has been used'
)
class Meta:
db_table = 'log_activity_tokens'
constraints = [
models.UniqueConstraint(fields=('version', 'user'), name='version_id'),
]
def is_expired(self):
return self.use_count >= MAX_TOKEN_USE_COUNT
def is_valid(self):
return (
not self.is_expired()
and self.version
== self.version.addon.find_latest_version(
channel=self.version.channel, exclude=()
)
)
def expire(self):
self.update(use_count=MAX_TOKEN_USE_COUNT)
def increment_use(self):
self.__class__.objects.filter(pk=self.pk).update(
use_count=models.expressions.F('use_count') + 1
)
self.use_count = self.use_count + 1
class CollectionAddon(ModelBase):
id = PositiveAutoField(primary_key=True)
addon = models.ForeignKey(Addon)
collection = models.ForeignKey(Collection)
# category (deprecated: for "Fashion Your Firefox")
comments = LinkifiedField(null=True)
user = models.ForeignKey(UserProfile, null=True)
ordering = models.PositiveIntegerField(
default=0,
help_text='Add-ons are displayed in ascending order '
'based on this field.')
class Meta(ModelBase.Meta):
db_table = 'addons_collections'
unique_together = (('addon', 'collection'), )
@staticmethod
def post_save(sender, instance, **kwargs):
"""Update Collection.addon_count and reindex add-on if the collection
is featured."""
from . import tasks
tasks.collection_meta.delay(instance.collection_id)
@staticmethod
def post_delete(sender, instance, **kwargs):
CollectionAddon.post_save(sender, instance, **kwargs)
if instance.collection.is_featured():
# The helpers .add_addon() and .remove_addon() already call .save()
# on the collection, triggering update_featured_status() among
# other things. However, this only takes care of the add-ons
# present in the collection at the time, we also need to make sure
# to invalidate add-ons that have been removed.
Collection.update_featured_status(sender,
instance.collection,
addons=[instance.addon.pk],
**kwargs)
class License(ModelBase):
OTHER = 0
id = PositiveAutoField(primary_key=True)
name = TranslatedField()
url = models.URLField(null=True)
builtin = models.PositiveIntegerField(default=OTHER)
text = LinkifiedField()
on_form = models.BooleanField(
default=False, help_text='Is this a license choice in the devhub?')
objects = LicenseManager()
class Meta:
db_table = 'licenses'
indexes = [models.Index(fields=('builtin', ), name='builtin_idx')]
def __str__(self):
license = self._constant or self
return str(license.name)
@property
def _constant(self):
return LICENSES_BY_BUILTIN.get(self.builtin)
@property
def creative_commons(self):
return bool((constant := self._constant) and constant.creative_commons)
@property
def icons(self):
return ((constant := self._constant) and constant.icons) or ''
@property
def slug(self):
return ((constant := self._constant) and constant.slug) or None
class AppVersion(ModelBase):
id = PositiveAutoField(primary_key=True)
application = models.PositiveIntegerField(choices=APPS_CHOICES,
db_column='application_id')
version = models.CharField(max_length=255, default='')
version_int = models.BigIntegerField(editable=False)
class Meta:
db_table = 'appversions'
ordering = ['-version_int']
unique_together = ('application', 'version')
def save(self, *args, **kw):
if not self.version_int:
self.version_int = compare.version_int(self.version)
return super(AppVersion, self).save(*args, **kw)
def __init__(self, *args, **kwargs):
super(AppVersion, self).__init__(*args, **kwargs)
# Add all the major, minor, ..., version attributes to the object.
self.__dict__.update(compare.version_dict(self.version or ''))
def __unicode__(self):
return self.version
class Version(OnChangeMixin, ModelBase):
id = PositiveAutoField(primary_key=True)
addon = models.ForeignKey('addons.Addon',
related_name='versions',
on_delete=models.CASCADE)
license = models.ForeignKey('License',
null=True,
blank=True,
on_delete=models.SET_NULL)
release_notes = PurifiedField(db_column='releasenotes', short=False)
approval_notes = models.TextField(db_column='approvalnotes',
default='',
null=True,
blank=True)
version = VersionStringField(max_length=255, default='0.1')
nomination = models.DateTimeField(null=True)
reviewed = models.DateTimeField(null=True)
deleted = models.BooleanField(default=False)
source = models.FileField(upload_to=source_upload_path,
null=True,
blank=True,
max_length=255)
channel = models.IntegerField(choices=amo.RELEASE_CHANNEL_CHOICES,
default=amo.RELEASE_CHANNEL_LISTED)
git_hash = models.CharField(max_length=40, blank=True)
needs_human_review = models.BooleanField(default=False)
# The order of those managers is very important: please read the lengthy
# comment above the Addon managers declaration/instantiation.
unfiltered = VersionManager(include_deleted=True)
objects = VersionManager()
# See UnfilteredVersionManagerForRelations() docstring for usage of this
# special manager.
unfiltered_for_relations = UnfilteredVersionManagerForRelations()
class Meta(ModelBase.Meta):
db_table = 'versions'
# This is very important: please read the lengthy comment in Addon.Meta
# description
base_manager_name = 'unfiltered'
ordering = ['-created', '-modified']
indexes = [
models.Index(fields=('addon', ), name='addon_id'),
models.Index(fields=('license', ), name='license_id'),
]
constraints = [
models.UniqueConstraint(
fields=('addon', 'version'),
name='versions_addon_id_version_5a2e75b6_uniq',
),
]
def __str__(self):
return markupsafe.escape(self.version)
@classmethod
def from_upload(cls,
upload,
addon,
selected_apps,
channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.addons.models import AddonReviewerFlags
from olympia.addons.utils import RestrictionChecker
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
if upload.addon and upload.addon != addon:
raise VersionCreateError(
'FileUpload was made for a different Addon')
if not upload.user or not upload.ip_address or not upload.source:
raise VersionCreateError(
'FileUpload does not have some required fields')
if not upload.user.last_login_ip or not upload.user.email:
raise VersionCreateError(
'FileUpload user does not have some required fields')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (
'This version has been signed with Mozilla internal certificate.'
)
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
# The following log statement is used by foxsec-pipeline.
# We override the IP because it might be called from a task and we
# want the original IP from the submitter.
log.info(
f'New version: {version!r} ({version.id}) from {upload!r}',
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
},
)
activity.log_create(amo.LOG.ADD_VERSION,
version,
addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# Pre-generate _compatible_apps property to avoid accidentally
# triggering queries with that instance later.
version._compatible_apps = compatible_apps
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [
File.from_upload(
upload=upload,
version=version,
parsed_data=parsed_data,
)
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to its permanent location, delete it
# from storage. Keep the FileUpload instance (it gets cleaned up by a
# cron eventually some time after its creation, in amo.cron.gc()),
# making sure it's associated with the add-on instance.
storage.delete(upload.path)
upload.path = ''
if upload.addon is None:
upload.addon = addon
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (waffle.switch_is_active('enable-yara')
or waffle.switch_is_active('enable-customs')
or waffle.switch_is_active('enable-wat')):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version))
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Reset add-on reviewer flags to disable auto-approval and require
# admin code review if the package has already been signed by mozilla.
reviewer_flags_defaults = {}
is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension')
if upload.validation_timeout:
reviewer_flags_defaults['needs_admin_code_review'] = True
if is_mozilla_signed and addon.type != amo.ADDON_LPAPP:
reviewer_flags_defaults['needs_admin_code_review'] = True
reviewer_flags_defaults['auto_approval_disabled'] = True
# Check if the approval should be restricted
if not RestrictionChecker(upload=upload).is_auto_approval_allowed():
flag = ('auto_approval_disabled'
if channel == amo.RELEASE_CHANNEL_LISTED else
'auto_approval_disabled_unlisted')
reviewer_flags_defaults[flag] = True
if reviewer_flags_defaults:
AddonReviewerFlags.objects.update_or_create(
addon=addon, defaults=reviewer_flags_defaults)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def license_url(self, impala=False):
return reverse('addons.license', args=[self.addon.slug, self.version])
def get_url_path(self):
if self.channel == amo.RELEASE_CHANNEL_UNLISTED:
return ''
return reverse('addons.versions', args=[self.addon.slug])
def delete(self, hard=False):
# To avoid a circular import
from .tasks import delete_preview_files
log.info(f'Version deleted: {self!r} ({self.id})')
activity.log_create(amo.LOG.DELETE_VERSION, self.addon,
str(self.version))
if hard:
super().delete()
else:
# By default we soft delete so we can keep the files for comparison
# and a record of the version number.
self.files.update(status=amo.STATUS_DISABLED)
self.deleted = True
self.save()
# Clear pending rejection flag (we have the activity log for
# records purposes, the flag serves no purpose anymore if the
# version is deleted).
VersionReviewerFlags.objects.filter(version=self).update(
pending_rejection=None)
previews_pks = list(
VersionPreview.objects.filter(version__id=self.id).values_list(
'id', flat=True))
for preview_pk in previews_pks:
delete_preview_files.delay(preview_pk)
@property
def is_user_disabled(self):
return (self.files.filter(status=amo.STATUS_DISABLED).exclude(
original_status=amo.STATUS_NULL).exists())
@is_user_disabled.setter
def is_user_disabled(self, disable):
# User wants to disable (and the File isn't already).
if disable:
activity.log_create(amo.LOG.DISABLE_VERSION, self.addon, self)
for file in self.files.exclude(status=amo.STATUS_DISABLED).all():
file.update(original_status=file.status,
status=amo.STATUS_DISABLED)
# User wants to re-enable (and user did the disable, not Mozilla).
else:
activity.log_create(amo.LOG.ENABLE_VERSION, self.addon, self)
for file in self.files.exclude(
original_status=amo.STATUS_NULL).all():
file.update(status=file.original_status,
original_status=amo.STATUS_NULL)
@cached_property
def all_activity(self):
# prefetch_related() and not select_related() the ActivityLog to make
# sure its transformer is called.
return self.versionlog_set.prefetch_related('activity_log').order_by(
'created')
@property
def compatible_apps(self):
# Dicts and search providers don't have compatibility info.
# Fake one for them.
if self.addon and self.addon.type in amo.NO_COMPAT:
return {app: None for app in amo.APP_TYPE_SUPPORT[self.addon.type]}
# Otherwise, return _compatible_apps which is a cached property that
# is filled by the transformer, or simply calculated from the related
# compat instances.
return self._compatible_apps
@cached_property
def _compatible_apps(self):
"""Get a mapping of {APP: ApplicationsVersions}."""
return self._compat_map(self.apps.all().select_related('min', 'max'))
@cached_property
def compatible_apps_ordered(self):
apps = self.compatible_apps.items()
return sorted(apps, key=lambda v: v[0].short)
@cached_property
def is_compatible_by_default(self):
"""Returns whether or not the add-on is considered compatible by
default."""
# Use self.all_files directly since that's cached and more potentially
# prefetched through a transformer already
return not any([
file for file in self.all_files
if file.binary_components or file.strict_compatibility
])
def is_compatible_app(self, app):
"""Returns True if the provided app passes compatibility conditions."""
if self.addon.type in amo.NO_COMPAT:
return True
appversion = self.compatible_apps.get(app)
if appversion and app.id in amo.D2C_MIN_VERSIONS:
return version_int(appversion.max.version) >= version_int(
amo.D2C_MIN_VERSIONS.get(app.id, '*'))
return False
def compat_override_app_versions(self):
"""Returns the incompatible app versions range(s).
If not ranges, returns empty list. Otherwise, this will return all
the app version ranges that this particular version is incompatible
with.
"""
overrides = list(self.addon.compatoverride_set.all())
if not overrides:
return []
app_versions = []
for co in overrides:
for range in co.collapsed_ranges():
if (version_int(range.min) <= version_int(self.version) <=
version_int(range.max)):
app_versions.extend([(a.min, a.max) for a in range.apps])
return app_versions
@cached_property
def all_files(self):
"""Shortcut for list(self.files.all()). Cached."""
return list(self.files.all())
@property
def current_file(self):
"""Shortcut for selecting the first file from self.all_files"""
return self.all_files[0]
@property
def status(self):
return [
f.STATUS_CHOICES.get(f.status,
gettext('[status:%s]') % f.status)
for f in self.all_files
]
@property
def statuses(self):
"""Unadulterated statuses, good for an API."""
return [(f.id, f.status) for f in self.all_files]
def is_public(self):
# To be public, a version must not be deleted, must belong to a public
# addon, and all its attached files must have public status.
try:
return (not self.deleted and self.addon.is_public()
and all(f.status == amo.STATUS_APPROVED
for f in self.all_files))
except ObjectDoesNotExist:
return False
@property
def is_webextension(self):
return any(file_.is_webextension for file_ in self.all_files)
@property
def is_mozilla_signed(self):
"""Is the file a special "Mozilla Signed Extension"
See https://wiki.mozilla.org/Add-ons/InternalSigning for more details.
We use that information to workaround compatibility limits for legacy
add-ons and to avoid them receiving negative boosts compared to
WebExtensions.
See https://github.com/mozilla/addons-server/issues/6424
"""
return all(file_.is_mozilla_signed_extension
for file_ in self.all_files)
@property
def has_files(self):
return bool(self.all_files)
@property
def is_unreviewed(self):
return bool(
list(
filter(lambda f: f.status in amo.UNREVIEWED_FILE_STATUSES,
self.all_files)))
@property
def is_all_unreviewed(self):
return not bool([
f for f in self.all_files
if f.status not in amo.UNREVIEWED_FILE_STATUSES
])
@property
def sources_provided(self):
return bool(self.source)
def _compat_map(self, avs):
apps = {}
for av in avs:
av.version = self
app_id = av.application
if app_id in amo.APP_IDS:
apps[amo.APP_IDS[app_id]] = av
return apps
@classmethod
def transformer(cls, versions):
"""Attach all the compatible apps and files to the versions."""
if not versions:
return
ids = {v.id for v in versions}
avs = ApplicationsVersions.objects.filter(
version__in=ids).select_related('min', 'max')
files = File.objects.filter(version__in=ids)
def rollup(xs):
groups = sorted_groupby(xs, 'version_id')
return {k: list(vs) for k, vs in groups}
av_dict, file_dict = rollup(avs), rollup(files)
for version in versions:
v_id = version.id
version._compatible_apps = version._compat_map(
av_dict.get(v_id, []))
version.all_files = file_dict.get(v_id, [])
for f in version.all_files:
f.version = version
@classmethod
def transformer_promoted(cls, versions):
"""Attach the promoted approvals to the versions."""
if not versions:
return
PromotedApproval = versions[0].promoted_approvals.model
ids = {v.id for v in versions}
approvals = list(
PromotedApproval.objects.filter(version_id__in=ids).values_list(
'version_id', 'group_id', 'application_id', named=True))
approval_dict = {
version_id: list(groups)
for version_id, groups in sorted_groupby(approvals, 'version_id')
}
for version in versions:
v_id = version.id
groups = [(
PROMOTED_GROUPS_BY_ID.get(approval.group_id),
APP_IDS.get(approval.application_id),
) for approval in approval_dict.get(v_id, [])
if approval.group_id in PROMOTED_GROUPS_BY_ID]
version.approved_for_groups = groups
@classmethod
def transformer_activity(cls, versions):
"""Attach all the activity to the versions."""
from olympia.activity.models import VersionLog
ids = {v.id for v in versions}
if not versions:
return
# Ideally, we'd start from the ActivityLog, but because VersionLog
# to ActivityLog isn't a OneToOneField, we wouldn't be able to find
# the version easily afterwards - we can't even do a
# select_related('versionlog') and try to traverse the relation to find
# the version. So, instead, start from VersionLog, but make sure to use
# prefetch_related() (and not select_related() - yes, it's one extra
# query, but it's worth it to benefit from the default transformer) so
# that the ActivityLog default transformer is called.
al = (VersionLog.objects.prefetch_related('activity_log').filter(
version__in=ids).order_by('created'))
def rollup(xs):
groups = sorted_groupby(xs, 'version_id')
return {k: list(vs) for k, vs in groups}
al_dict = rollup(al)
for version in versions:
v_id = version.id
version.all_activity = al_dict.get(v_id, [])
@classmethod
def transformer_license(cls, versions):
"""Attach all the licenses to the versions.
Do not use if you need the license text: it's explicitly deferred in
this transformer, because it should only be used when listing multiple
versions, where returning license text is not supposed to be needed.
The translations app doesn't fully handle evaluating a deferred field,
so the callers need to make sure the license text will never be needed
on instances returned by a queryset transformed by this method."""
if not versions:
return
license_ids = {ver.license_id for ver in versions}
licenses = License.objects.filter(id__in=license_ids).defer('text')
license_dict = {lic.id: lic for lic in licenses}
for version in versions:
license = license_dict.get(version.license_id)
if license:
version.license = license
@classmethod
def transformer_auto_approvable(cls, versions):
"""Attach auto-approvability information to the versions."""
ids = {v.id for v in versions}
if not ids:
return
auto_approvable = (Version.objects.auto_approvable().filter(
pk__in=ids).values_list('pk', flat=True))
for version in versions:
version.is_ready_for_auto_approval = version.pk in auto_approvable
def disable_old_files(self):
"""
Disable files from versions older than the current one in the same
channel and awaiting review. Used when uploading a new version.
Does nothing if the current instance is unlisted.
"""
if self.channel == amo.RELEASE_CHANNEL_LISTED:
qs = File.objects.filter(
version__addon=self.addon_id,
version__lt=self.id,
version__deleted=False,
version__channel=self.channel,
status=amo.STATUS_AWAITING_REVIEW,
)
# Use File.update so signals are triggered.
for f in qs:
f.update(status=amo.STATUS_DISABLED)
def reset_nomination_time(self, nomination=None):
if not self.nomination or nomination:
nomination = nomination or datetime.datetime.now()
# We need signal=False not to call update_status (which calls us).
self.update(nomination=nomination, _signal=False)
def inherit_nomination(self, from_statuses=None):
last_ver = (Version.objects.filter(
addon=self.addon, channel=amo.RELEASE_CHANNEL_LISTED).exclude(
nomination=None).exclude(id=self.pk).order_by('-nomination'))
if from_statuses:
last_ver = last_ver.filter(files__status__in=from_statuses)
if last_ver.exists():
self.reset_nomination_time(nomination=last_ver[0].nomination)
@property
def unreviewed_files(self):
"""A File is unreviewed if its status is amo.STATUS_AWAITING_REVIEW."""
return self.files.filter(status=amo.STATUS_AWAITING_REVIEW)
@cached_property
def is_ready_for_auto_approval(self):
"""Return whether or not this version could be *considered* for
auto-approval.
Does not necessarily mean that it would be auto-approved, just that it
passes the most basic criteria to be considered a candidate by the
auto_approve command."""
return Version.objects.auto_approvable().filter(id=self.id).exists()
@property
def was_auto_approved(self):
"""Return whether or not this version was auto-approved."""
from olympia.reviewers.models import AutoApprovalSummary
try:
return (self.is_public() and AutoApprovalSummary.objects.filter(
version=self).get().verdict == amo.AUTO_APPROVED)
except AutoApprovalSummary.DoesNotExist:
pass
return False
def get_background_images_encoded(self, header_only=False):
if not self.has_files:
return {}
file_obj = self.all_files[0]
return {
name: force_str(b64encode(background))
for name, background in utils.get_background_images(
file_obj, theme_data=None, header_only=header_only).items()
}
def can_be_disabled_and_deleted(self):
# see https://github.com/mozilla/addons-server/issues/15121#issuecomment-667226959 # noqa
# "It should apply to the <groups> that require a review to be badged"
from olympia.promoted.models import PromotedApproval
if self != self.addon.current_version or (
not (group := self.addon.promoted_group())
or not (group.badged and group.pre_review)):
return True
previous_ver = (self.addon.versions.valid().filter(
channel=self.channel).exclude(id=self.id).no_transforms()[:1])
previous_approval = PromotedApproval.objects.filter(
group_id=group.id, version__in=previous_ver)
return previous_approval.exists()
class CollectionAddon(ModelBase):
id = PositiveAutoField(primary_key=True)
addon = models.ForeignKey(Addon, on_delete=models.CASCADE)
collection = models.ForeignKey(Collection, on_delete=models.CASCADE)
# category (deprecated: for "Fashion Your Firefox")
comments = LinkifiedField(null=True)
user = models.ForeignKey(UserProfile, null=True, on_delete=models.CASCADE)
ordering = models.PositiveIntegerField(
default=0,
help_text='Add-ons are displayed in ascending order based on this field.',
)
class Meta(ModelBase.Meta):
db_table = 'addons_collections'
indexes = [
models.Index(
fields=('collection', 'created'), name='addons_collections_created_idx'
),
models.Index(fields=('addon',), name='addons_collections_addon_idx'),
models.Index(fields=('collection',), name='collection_id'),
models.Index(fields=('user',), name='addons_collections_user_id'),
]
constraints = [
models.UniqueConstraint(fields=('addon', 'collection'), name='addon_id_2'),
]
@staticmethod
def post_save(sender, instance, **kwargs):
"""Update Collection.addon_count and reindex add-on if the collection
is featured."""
from olympia.addons.tasks import index_addons
if kwargs.get('raw'):
return
if instance.collection.listed:
activity.log_create(
amo.LOG.ADD_TO_COLLECTION, instance.addon, instance.collection
)
kwargs['addons'] = [instance.addon]
Collection.post_save(sender, instance.collection, **kwargs)
if instance.collection.id == settings.COLLECTION_FEATURED_THEMES_ID:
# That collection is special: each add-on in it is considered
# recommended, so we need to index the corresponding add-on.
# (Note: we are considering the add-on in a given CollectionAddon
# never changes, to change add-ons belonging to a collection we
# add or remove CollectionAddon instances, we never modify the
# addon foreignkey of an existing instance).
index_addons.delay([instance.addon.id])
@staticmethod
def post_delete(sender, instance, **kwargs):
from olympia.addons.tasks import index_addons
if kwargs.get('raw'):
return
if instance.collection.listed:
activity.log_create(
amo.LOG.REMOVE_FROM_COLLECTION, instance.addon, instance.collection
)
kwargs['addons'] = [instance.addon]
Collection.post_save(sender, instance.collection, **kwargs)
if instance.collection.id == settings.COLLECTION_FEATURED_THEMES_ID:
# That collection is special: each add-on in it is considered
# recommended, so we need to index the add-on we just removed from
# it.
index_addons.delay([instance.addon.id])
class ReviewerScore(ModelBase):
id = PositiveAutoField(primary_key=True)
user = models.ForeignKey(UserProfile, related_name='_reviewer_scores')
addon = models.ForeignKey(Addon, blank=True, null=True, related_name='+')
version = models.ForeignKey(Version,
blank=True,
null=True,
related_name='+')
score = models.IntegerField()
# For automated point rewards.
note_key = models.SmallIntegerField(choices=amo.REVIEWED_CHOICES.items(),
default=0)
# For manual point rewards with a note.
note = models.CharField(max_length=255)
class Meta:
db_table = 'reviewer_scores'
ordering = ('-created', )
@classmethod
def get_key(cls, key=None, invalidate=False):
namespace = 'riscore'
if not key: # Assuming we're invalidating the namespace.
cache_ns_key(namespace, invalidate)
return
else:
# Using cache_ns_key so each cache val is invalidated together.
ns_key = cache_ns_key(namespace, invalidate)
return '%s:%s' % (ns_key, key)
@classmethod
def get_event(cls,
addon,
status,
version=None,
post_review=False,
content_review=False):
"""Return the review event type constant.
This is determined by the addon.type and the queue the addon is
currently in (which is determined from the various parameters sent
down from award_points()).
Note: We're not using addon.status or addon.current_version because
this is called after the status/current_version might have been updated
by the reviewer action.
"""
reviewed_score_name = None
if content_review:
# Content review always gives the same amount of points.
reviewed_score_name = 'REVIEWED_CONTENT_REVIEW'
elif post_review:
# There are 4 tiers of post-review scores depending on the addon
# weight.
try:
if version is None:
raise AutoApprovalSummary.DoesNotExist
weight = version.autoapprovalsummary.weight
except AutoApprovalSummary.DoesNotExist as exception:
log.exception(
'No such version/auto approval summary when determining '
'event type to award points: %r', exception)
weight = 0
if addon.type == amo.ADDON_DICT:
reviewed_score_name = 'REVIEWED_DICT_FULL'
elif addon.type in [amo.ADDON_LPAPP, amo.ADDON_LPADDON]:
reviewed_score_name = 'REVIEWED_LP_FULL'
elif addon.type == amo.ADDON_SEARCH:
reviewed_score_name = 'REVIEWED_SEARCH_FULL'
elif weight > amo.POST_REVIEW_WEIGHT_HIGHEST_RISK:
reviewed_score_name = 'REVIEWED_EXTENSION_HIGHEST_RISK'
elif weight > amo.POST_REVIEW_WEIGHT_HIGH_RISK:
reviewed_score_name = 'REVIEWED_EXTENSION_HIGH_RISK'
elif weight > amo.POST_REVIEW_WEIGHT_MEDIUM_RISK:
reviewed_score_name = 'REVIEWED_EXTENSION_MEDIUM_RISK'
else:
reviewed_score_name = 'REVIEWED_EXTENSION_LOW_RISK'
else:
if status == amo.STATUS_NOMINATED:
queue = 'FULL'
elif status == amo.STATUS_PUBLIC:
queue = 'UPDATE'
else:
queue = ''
if (addon.type
in [amo.ADDON_EXTENSION, amo.ADDON_PLUGIN, amo.ADDON_API]
and queue):
reviewed_score_name = 'REVIEWED_ADDON_%s' % queue
elif addon.type == amo.ADDON_DICT and queue:
reviewed_score_name = 'REVIEWED_DICT_%s' % queue
elif addon.type in [amo.ADDON_LPAPP, amo.ADDON_LPADDON] and queue:
reviewed_score_name = 'REVIEWED_LP_%s' % queue
elif addon.type == amo.ADDON_PERSONA:
reviewed_score_name = 'REVIEWED_PERSONA'
elif addon.type == amo.ADDON_STATICTHEME:
reviewed_score_name = 'REVIEWED_STATICTHEME'
elif addon.type == amo.ADDON_SEARCH and queue:
reviewed_score_name = 'REVIEWED_SEARCH_%s' % queue
elif addon.type == amo.ADDON_THEME and queue:
reviewed_score_name = 'REVIEWED_XUL_THEME_%s' % queue
if reviewed_score_name:
return getattr(amo, reviewed_score_name)
return None
@classmethod
def award_points(cls,
user,
addon,
status,
version=None,
post_review=False,
content_review=False,
extra_note=''):
"""Awards points to user based on an event and the queue.
`event` is one of the `REVIEWED_` keys in constants.
`status` is one of the `STATUS_` keys in constants.
`version` is the `Version` object that was affected by the review.
`post_review` is set to True if the add-on was auto-approved and the
reviewer is confirming/rejecting post-approval.
`content_review` is set to True if it's a content-only review of an
auto-approved add-on.
"""
# If a webextension file gets approved manually (e.g. because
# auto-approval is disabled), 'post-review' is set to False, treating
# the file as a legacy file which is not what we want. The file is
# still a webextension and should treated as such, regardless of
# auto-approval being disabled or not.
# As a hack, we set 'post_review' to True.
if (version and version.is_webextension
and addon.type in amo.GROUP_TYPE_ADDON):
post_review = True
user_log.info(
(u'Determining award points for user %s for version %s of addon %s'
% (user, version, addon.id)).encode('utf-8'))
event = cls.get_event(addon,
status,
version=version,
post_review=post_review,
content_review=content_review)
score = amo.REVIEWED_SCORES.get(event)
user_log.info(
(u'Determined %s award points (event: %s) for user %s for version '
u'%s of addon %s' %
(score, event, user, version, addon.id)).encode('utf-8'))
# Add bonus to reviews greater than our limit to encourage fixing
# old reviews. Does not apply to content-review/post-review at the
# moment, because it would need to be calculated differently.
award_overdue_bonus = (version and version.nomination
and not post_review and not content_review)
if award_overdue_bonus:
waiting_time_days = (datetime.now() - version.nomination).days
days_over = waiting_time_days - amo.REVIEWED_OVERDUE_LIMIT
if days_over > 0:
bonus = days_over * amo.REVIEWED_OVERDUE_BONUS
score = score + bonus
if score is not None:
cls.objects.create(user=user,
addon=addon,
score=score,
note_key=event,
note=extra_note,
version=version)
cls.get_key(invalidate=True)
user_log.info(
(u'Awarding %s points to user %s for "%s" for addon %s' %
(score, user, amo.REVIEWED_CHOICES[event],
addon.id)).encode('utf-8'))
return score
@classmethod
def award_moderation_points(cls, user, addon, review_id, undo=False):
"""Awards points to user based on moderated review."""
event = (amo.REVIEWED_ADDON_REVIEW
if not undo else amo.REVIEWED_ADDON_REVIEW_POORLY)
score = amo.REVIEWED_SCORES.get(event)
cls.objects.create(user=user, addon=addon, score=score, note_key=event)
cls.get_key(invalidate=True)
user_log.info(u'Awarding %s points to user %s for "%s" for review %s' %
(score, user, amo.REVIEWED_CHOICES[event], review_id))
@classmethod
def get_total(cls, user):
"""Returns total points by user."""
key = cls.get_key('get_total:%s' % user.id)
val = cache.get(key)
if val is not None:
return val
val = (ReviewerScore.objects.filter(user=user).aggregate(
total=Sum('score')).values())[0]
if val is None:
val = 0
cache.set(key, val, None)
return val
@classmethod
def get_recent(cls, user, limit=5, addon_type=None):
"""Returns most recent ReviewerScore records."""
key = cls.get_key('get_recent:%s' % user.id)
val = cache.get(key)
if val is not None:
return val
val = ReviewerScore.objects.filter(user=user)
if addon_type is not None:
val.filter(addon__type=addon_type)
val = list(val[:limit])
cache.set(key, val, None)
return val
@classmethod
def get_breakdown(cls, user):
"""Returns points broken down by addon type."""
key = cls.get_key('get_breakdown:%s' % user.id)
val = cache.get(key)
if val is not None:
return val
sql = """
SELECT `reviewer_scores`.*,
SUM(`reviewer_scores`.`score`) AS `total`,
`addons`.`addontype_id` AS `atype`
FROM `reviewer_scores`
LEFT JOIN `addons` ON (`reviewer_scores`.`addon_id`=`addons`.`id`)
WHERE `reviewer_scores`.`user_id` = %s
GROUP BY `addons`.`addontype_id`
ORDER BY `total` DESC
"""
val = list(ReviewerScore.objects.raw(sql, [user.id]))
cache.set(key, val, None)
return val
@classmethod
def get_breakdown_since(cls, user, since):
"""
Returns points broken down by addon type since the given datetime.
"""
key = cls.get_key('get_breakdown:%s:%s' % (user.id, since.isoformat()))
val = cache.get(key)
if val is not None:
return val
sql = """
SELECT `reviewer_scores`.*,
SUM(`reviewer_scores`.`score`) AS `total`,
`addons`.`addontype_id` AS `atype`
FROM `reviewer_scores`
LEFT JOIN `addons` ON (`reviewer_scores`.`addon_id`=`addons`.`id`)
WHERE `reviewer_scores`.`user_id` = %s AND
`reviewer_scores`.`created` >= %s
GROUP BY `addons`.`addontype_id`
ORDER BY `total` DESC
"""
val = list(ReviewerScore.objects.raw(sql, [user.id, since]))
cache.set(key, val, 3600)
return val
@classmethod
def _leaderboard_list(cls, since=None, types=None, addon_type=None):
"""
Returns base leaderboard list. Each item will be a tuple containing
(user_id, name, total).
"""
reviewers = (UserProfile.objects.filter(
groups__name__startswith='Reviewers: ').exclude(
groups__name__in=('Staff', 'Admins',
'No Reviewer Incentives')).distinct())
qs = (cls.objects.values_list('user__id').filter(
user__in=reviewers).annotate(
total=Sum('score')).order_by('-total'))
if since is not None:
qs = qs.filter(created__gte=since)
if types is not None:
qs = qs.filter(note_key__in=types)
if addon_type is not None:
qs = qs.filter(addon__type=addon_type)
users = {reviewer.pk: reviewer for reviewer in reviewers}
return [(item[0], users.get(item[0], UserProfile()).name, item[1])
for item in qs]
@classmethod
def get_leaderboards(cls, user, days=7, types=None, addon_type=None):
"""Returns leaderboards with ranking for the past given days.
This will return a dict of 3 items::
{'leader_top': [...],
'leader_near: [...],
'user_rank': (int)}
If the user is not in the leaderboard, or if the user is in the top 5,
'leader_near' will be an empty list and 'leader_top' will contain 5
elements instead of the normal 3.
"""
key = cls.get_key('get_leaderboards:%s' % user.id)
val = cache.get(key)
if val is not None:
return val
week_ago = date.today() - timedelta(days=days)
leader_top = []
leader_near = []
leaderboard = cls._leaderboard_list(since=week_ago,
types=types,
addon_type=addon_type)
scores = []
user_rank = 0
in_leaderboard = False
for rank, row in enumerate(leaderboard, 1):
user_id, name, total = row
scores.append({
'user_id': user_id,
'name': name,
'rank': rank,
'total': int(total),
})
if user_id == user.id:
user_rank = rank
in_leaderboard = True
if not in_leaderboard:
leader_top = scores[:5]
else:
if user_rank <= 5: # User is in top 5, show top 5.
leader_top = scores[:5]
else:
leader_top = scores[:3]
leader_near = [scores[user_rank - 2], scores[user_rank - 1]]
try:
leader_near.append(scores[user_rank])
except IndexError:
pass # User is last on the leaderboard.
val = {
'leader_top': leader_top,
'leader_near': leader_near,
'user_rank': user_rank,
}
cache.set(key, val, None)
return val
@classmethod
def all_users_by_score(cls):
"""
Returns reviewers ordered by highest total points first.
"""
leaderboard = cls._leaderboard_list()
scores = []
for row in leaderboard:
user_id, name, total = row
user_level = len(amo.REVIEWED_LEVELS) - 1
for i, level in enumerate(amo.REVIEWED_LEVELS):
if total < level['points']:
user_level = i - 1
break
# Only show level if it changes.
if user_level < 0:
level = ''
else:
level = unicode(amo.REVIEWED_LEVELS[user_level]['name'])
scores.append({
'user_id': user_id,
'name': name,
'total': int(total),
'level': level,
})
prev = None
for score in reversed(scores):
if score['level'] == prev:
score['level'] = ''
else:
prev = score['level']
return scores
class File(OnChangeMixin, ModelBase):
id = PositiveAutoField(primary_key=True)
STATUS_CHOICES = amo.STATUS_CHOICES_FILE
version = models.ForeignKey('versions.Version',
related_name='files',
on_delete=models.CASCADE)
platform = models.PositiveIntegerField(
choices=amo.SUPPORTED_PLATFORMS_CHOICES,
default=amo.PLATFORM_ALL.id,
db_column="platform_id")
filename = models.CharField(max_length=255, default='')
size = models.PositiveIntegerField(default=0) # In bytes.
hash = models.CharField(max_length=255, default='')
# The original hash of the file, before we sign it, or repackage it in
# any other way.
original_hash = models.CharField(max_length=255, default='')
status = models.PositiveSmallIntegerField(
choices=STATUS_CHOICES.items(), default=amo.STATUS_AWAITING_REVIEW)
datestatuschanged = models.DateTimeField(null=True, auto_now_add=True)
is_restart_required = models.BooleanField(default=False)
strict_compatibility = models.BooleanField(default=False)
reviewed = models.DateTimeField(null=True, blank=True)
# The `binary` field is used to store the flags from amo-validator when it
# finds files with binary extensions or files that may contain binary
# content.
binary = models.BooleanField(default=False)
# The `binary_components` field is used to store the flag from
# amo-validator when it finds "binary-components" in the chrome manifest
# file, used for default to compatible.
binary_components = models.BooleanField(default=False, db_index=True)
# Serial number of the certificate use for the signature.
cert_serial_num = models.TextField(blank=True)
# Is the file signed by Mozilla?
is_signed = models.BooleanField(default=False)
# Is the file an experiment (see bug 1220097)?
is_experiment = models.BooleanField(default=False)
# Is the file a WebExtension?
is_webextension = models.BooleanField(default=False)
# Is the file a special "Mozilla Signed Extension"
# see https://wiki.mozilla.org/Add-ons/InternalSigning
is_mozilla_signed_extension = models.BooleanField(default=False)
# The user has disabled this file and this was its status.
# STATUS_NULL means the user didn't disable the File - i.e. Mozilla did.
original_status = models.PositiveSmallIntegerField(default=amo.STATUS_NULL)
class Meta(ModelBase.Meta):
db_table = 'files'
def __str__(self):
return six.text_type(self.id)
def get_platform_display(self):
return force_text(amo.PLATFORMS[self.platform].name)
@property
def has_been_validated(self):
try:
self.validation
except FileValidation.DoesNotExist:
return False
else:
return True
@property
def automated_signing(self):
"""True if this file is eligible for automated signing. This currently
means that either its version is unlisted."""
return self.version.channel == amo.RELEASE_CHANNEL_UNLISTED
def get_file_cdn_url(self, attachment=False):
"""Return the URL for the file corresponding to this instance
on the CDN."""
if attachment:
host = posixpath.join(user_media_url('addons'), '_attachments')
else:
host = user_media_url('addons')
return posixpath.join(
*map(force_bytes, [host, self.version.addon.id, self.filename]))
def get_url_path(self, src, attachment=False):
return self._make_download_url('downloads.file',
src,
attachment=attachment)
def _make_download_url(self, view_name, src, attachment=False):
kwargs = {'file_id': self.pk}
if attachment:
kwargs['type'] = 'attachment'
url = os.path.join(reverse(view_name, kwargs=kwargs), self.filename)
return absolutify(urlparams(url, src=src))
@classmethod
def from_upload(cls, upload, version, platform, parsed_data=None):
"""
Create a File instance from a FileUpload, a Version, a platform id
and the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results."""
assert parsed_data is not None
file_ = cls(version=version, platform=platform)
upload_path = force_text(nfd_str(upload.path))
ext = force_text(os.path.splitext(upload_path)[1])
if ext == '.jar':
ext = '.xpi'
file_.filename = file_.generate_filename(extension=ext or '.xpi')
# Size in bytes.
file_.size = storage.size(upload_path)
file_.is_restart_required = parsed_data.get('is_restart_required',
False)
file_.strict_compatibility = parsed_data.get('strict_compatibility',
False)
file_.is_experiment = parsed_data.get('is_experiment', False)
file_.is_webextension = parsed_data.get('is_webextension', False)
file_.is_mozilla_signed_extension = parsed_data.get(
'is_mozilla_signed_extension', False)
file_.hash = file_.generate_hash(upload_path)
file_.original_hash = file_.hash
file_.save()
if file_.is_webextension:
permissions = list(parsed_data.get('permissions', []))
# Add content_scripts host matches too.
for script in parsed_data.get('content_scripts', []):
permissions.extend(script.get('matches', []))
if permissions:
WebextPermission.objects.create(permissions=permissions,
file=file_)
log.debug('New file: %r from %r' % (file_, upload))
# Move the uploaded file from the temp location.
copy_stored_file(upload_path, file_.current_file_path)
if upload.validation:
validation = json.loads(upload.validation)
FileValidation.from_json(file_, validation)
return file_
def generate_hash(self, filename=None):
"""Generate a hash for a file."""
with open(filename or self.current_file_path, 'rb') as fobj:
return 'sha256:{}'.format(get_sha256(fobj))
def generate_filename(self, extension=None):
"""
Files are in the format of:
{addon_name}-{version}-{apps}-{platform}
"""
parts = []
addon = self.version.addon
# slugify drops unicode so we may end up with an empty string.
# Apache did not like serving unicode filenames (bug 626587).
extension = extension or '.xpi'
name = slugify(addon.name).replace('-', '_') or 'addon'
parts.append(name)
parts.append(self.version.version)
if addon.type not in amo.NO_COMPAT and self.version.compatible_apps:
apps = '+'.join(
sorted([a.shortername for a in self.version.compatible_apps]))
parts.append(apps)
if self.platform and self.platform != amo.PLATFORM_ALL.id:
parts.append(amo.PLATFORMS[self.platform].shortname)
self.filename = '-'.join(parts) + extension
return self.filename
_pretty_filename = re.compile(r'(?P<slug>[a-z0-7_]+)(?P<suffix>.*)')
def pretty_filename(self, maxlen=20):
"""Displayable filename.
Truncates filename so that the slug part fits maxlen.
"""
m = self._pretty_filename.match(self.filename)
if not m:
return self.filename
if len(m.group('slug')) < maxlen:
return self.filename
return u'%s...%s' % (m.group('slug')[0:(maxlen - 3)],
m.group('suffix'))
def latest_xpi_url(self, attachment=False):
addon = self.version.addon
kw = {'addon_id': addon.slug}
if self.platform != amo.PLATFORM_ALL.id:
kw['platform'] = self.platform
if attachment:
kw['type'] = 'attachment'
return os.path.join(reverse('downloads.latest', kwargs=kw),
'addon-%s-latest%s' % (addon.pk, self.extension))
@property
def file_path(self):
return os.path.join(user_media_path('addons'),
str(self.version.addon_id), self.filename)
@property
def addon(self):
return self.version.addon
@property
def guarded_file_path(self):
return os.path.join(user_media_path('guarded_addons'),
str(self.version.addon_id), self.filename)
@property
def current_file_path(self):
"""Returns the current path of the file, whether or not it is
guarded."""
file_disabled = self.status == amo.STATUS_DISABLED
addon_disabled = self.addon.is_disabled
if file_disabled or addon_disabled:
return self.guarded_file_path
else:
return self.file_path
@property
def extension(self):
return os.path.splitext(self.filename)[-1]
def move_file(self, source, destination, log_message):
"""Move a file from `source` to `destination`."""
log_message = force_text(log_message)
try:
if storage.exists(source):
log.info(
log_message.format(source=source, destination=destination))
move_stored_file(source, destination)
except (UnicodeEncodeError, IOError):
msg = u'Move Failure: {} {}'.format(source, destination)
log.exception(msg)
def hide_disabled_file(self):
"""Move a disabled file to the guarded file path."""
if not self.filename:
return
src, dst = self.file_path, self.guarded_file_path
self.move_file(src, dst,
'Moving disabled file: {source} => {destination}')
def unhide_disabled_file(self):
if not self.filename:
return
src, dst = self.guarded_file_path, self.file_path
self.move_file(src, dst,
'Moving undisabled file: {source} => {destination}')
_get_localepicker = re.compile(r'^locale browser ([\w\-_]+) (.*)$', re.M)
@memoize(prefix='localepicker', timeout=None)
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
try:
zip_ = SafeZip(self.file_path)
except (zipfile.BadZipfile, IOError):
return ''
try:
manifest = force_text(zip_.read('chrome.manifest'))
except KeyError:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
path = res.groups()[1]
if 'localepicker.properties' not in path:
path = os.path.join(path, 'localepicker.properties')
res = zip_.extract_from_manifest(path)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' %
(path, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' % (self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return force_text(res)
@cached_property
def webext_permissions_list(self):
if not self.is_webextension:
return []
try:
# Filter out any errant non-strings included in the manifest JSON.
# Remove any duplicate permissions.
permissions = set()
permissions = [
p for p in self._webext_permissions.permissions
if isinstance(p, six.string_types)
and not (p in permissions or permissions.add(p))
]
return permissions
except WebextPermission.DoesNotExist:
return []
class Collection(ModelBase):
id = PositiveAutoField(primary_key=True)
TYPE_CHOICES = amo.COLLECTION_CHOICES.items()
uuid = models.UUIDField(blank=True, unique=True, null=True)
name = TranslatedField(require_locale=False)
# nickname is deprecated. Use slug.
nickname = models.CharField(max_length=30,
blank=True,
unique=True,
null=True)
slug = models.CharField(max_length=30, blank=True, null=True)
description = NoLinksNoMarkupField(require_locale=False)
default_locale = models.CharField(max_length=10,
default='en-US',
db_column='defaultlocale')
type = models.PositiveIntegerField(db_column='collection_type',
choices=TYPE_CHOICES,
default=0)
listed = models.BooleanField(
default=True, help_text='Collections are either listed or private.')
application = models.PositiveIntegerField(choices=amo.APPS_CHOICES,
db_column='application_id',
blank=True,
null=True,
db_index=True)
addon_count = models.PositiveIntegerField(default=0,
db_column='addonCount')
all_personas = models.BooleanField(
default=False, help_text='Does this collection only contain Themes?')
addons = models.ManyToManyField(Addon,
through='CollectionAddon',
related_name='collections')
author = models.ForeignKey(UserProfile,
null=True,
related_name='collections')
objects = CollectionManager()
class Meta(ModelBase.Meta):
db_table = 'collections'
unique_together = (('author', 'slug'), )
def __str__(self):
return u'%s (%s)' % (self.name, self.addon_count)
def save(self, **kw):
if not self.uuid:
self.uuid = uuid.uuid4()
if not self.slug:
# Work with both, strings (if passed manually on .create()
# and UUID instances)
self.slug = str(self.uuid).replace('-', '')[:30]
self.clean_slug()
super(Collection, self).save(**kw)
def clean_slug(self):
if self.type in SPECIAL_SLUGS:
self.slug = SPECIAL_SLUGS[self.type]
return
if self.slug in SPECIAL_SLUGS.values():
self.slug += '~'
if not self.author:
return
qs = self.author.collections.using('default')
slugs = dict((slug, id) for slug, id in qs.values_list('slug', 'id'))
if self.slug in slugs and slugs[self.slug] != self.id:
for idx in range(len(slugs)):
new = '%s-%s' % (self.slug, idx + 1)
if new not in slugs:
self.slug = new
return
def get_url_path(self):
return reverse('collections.detail', args=[self.author_id, self.slug])
def get_abs_url(self):
return absolutify(self.get_url_path())
def edit_url(self):
return reverse('collections.edit', args=[self.author_id, self.slug])
def delete_url(self):
return reverse('collections.delete', args=[self.author_id, self.slug])
def share_url(self):
return reverse('collections.share', args=[self.author_id, self.slug])
def stats_url(self):
return reverse('collections.stats', args=[self.author_id, self.slug])
@classmethod
def get_fallback(cls):
return cls._meta.get_field('default_locale')
def set_addons(self, addon_ids, comments=None):
"""Replace the current add-ons with a new list of add-on ids."""
if comments is None:
comments = {}
order = {a: idx for idx, a in enumerate(addon_ids)}
# Partition addon_ids into add/update/remove buckets.
existing = set(
self.addons.using('default').values_list('id', flat=True))
add, update = [], []
for addon in addon_ids:
bucket = update if addon in existing else add
bucket.append((addon, order[addon]))
remove = existing.difference(addon_ids)
now = datetime.now()
with connection.cursor() as cursor:
if remove:
cursor.execute("DELETE FROM addons_collections "
"WHERE collection_id=%s AND addon_id IN (%s)" %
(self.id, ','.join(map(str, remove))))
if self.listed:
for addon in remove:
activity.log_create(amo.LOG.REMOVE_FROM_COLLECTION,
(Addon, addon), self)
if add:
insert = '(%s, %s, %s, NOW(), NOW())'
values = [insert % (a, self.id, idx) for a, idx in add]
cursor.execute("""
INSERT INTO addons_collections
(addon_id, collection_id, ordering, created, modified)
VALUES %s""" % ','.join(values))
if self.listed:
for addon_id, idx in add:
activity.log_create(amo.LOG.ADD_TO_COLLECTION,
(Addon, addon_id), self)
for addon, ordering in update:
(CollectionAddon.objects.filter(
collection=self.id, addon=addon).update(ordering=ordering,
modified=now))
for addon, comment in six.iteritems(comments):
try:
c = (CollectionAddon.objects.using('default').get(
collection=self.id, addon=addon))
except CollectionAddon.DoesNotExist:
pass
else:
c.comments = comment
c.save(force_update=True)
self.save()
def add_addon(self, addon):
"Adds an addon to the collection."
CollectionAddon.objects.get_or_create(addon=addon, collection=self)
if self.listed:
activity.log_create(amo.LOG.ADD_TO_COLLECTION, addon, self)
self.save() # To invalidate Collection.
def remove_addon(self, addon):
CollectionAddon.objects.filter(addon=addon, collection=self).delete()
if self.listed:
activity.log_create(amo.LOG.REMOVE_FROM_COLLECTION, addon, self)
self.save() # To invalidate Collection.
def owned_by(self, user):
return user.id == self.author_id
def can_view_stats(self, request):
if request and request.user:
return (self.owned_by(request.user) or acl.action_allowed(
request, amo.permissions.COLLECTION_STATS_VIEW))
return False
def is_public(self):
return self.listed
def is_featured(self):
return FeaturedCollection.objects.filter(collection=self).exists()
@staticmethod
def transformer(collections):
if not collections:
return
author_ids = set(c.author_id for c in collections)
authors = dict(
(u.id, u) for u in UserProfile.objects.filter(id__in=author_ids))
for c in collections:
c.author = authors.get(c.author_id)
@staticmethod
def post_save(sender, instance, **kwargs):
from . import tasks
if kwargs.get('raw'):
return
tasks.collection_meta.delay(instance.id)
if instance.is_featured():
Collection.update_featured_status(sender, instance, **kwargs)
@staticmethod
def post_delete(sender, instance, **kwargs):
if kwargs.get('raw'):
return
if instance.is_featured():
Collection.update_featured_status(sender, instance, **kwargs)
@staticmethod
def update_featured_status(sender, instance, **kwargs):
from olympia.addons.tasks import index_addons
addons = kwargs.get('addons',
[addon.id for addon in instance.addons.all()])
if addons:
clear_get_featured_ids_cache(None, None)
index_addons.delay(addons)
def check_ownership(self, request, require_owner, require_author,
ignore_disabled, admin):
"""
Used by acl.check_ownership to see if request.user has permissions for
the collection.
"""
from olympia.access import acl
return acl.check_collection_ownership(request, self, require_owner)
class File(OnChangeMixin, ModelBase):
id = PositiveAutoField(primary_key=True)
STATUS_CHOICES = amo.STATUS_CHOICES_FILE
version = models.ForeignKey('versions.Version',
related_name='files',
on_delete=models.CASCADE)
filename = models.CharField(max_length=255, default='')
size = models.PositiveIntegerField(default=0) # In bytes.
hash = models.CharField(max_length=255, default='')
# The original hash of the file, before we sign it, or repackage it in
# any other way.
original_hash = models.CharField(max_length=255, default='')
status = models.PositiveSmallIntegerField(
choices=STATUS_CHOICES.items(), default=amo.STATUS_AWAITING_REVIEW)
datestatuschanged = models.DateTimeField(null=True, auto_now_add=True)
is_restart_required = models.BooleanField(default=False)
strict_compatibility = models.BooleanField(default=False)
reviewed = models.DateTimeField(null=True, blank=True)
# The `binary` field is used to store the flags from amo-validator when it
# finds files with binary extensions or files that may contain binary
# content.
binary = models.BooleanField(default=False)
# The `binary_components` field is used to store the flag from
# amo-validator when it finds "binary-components" in the chrome manifest
# file, used for default to compatible.
binary_components = models.BooleanField(default=False)
# Serial number of the certificate use for the signature.
cert_serial_num = models.TextField(blank=True)
# Is the file signed by Mozilla?
is_signed = models.BooleanField(default=False)
# Is the file an experiment (see bug 1220097)?
is_experiment = models.BooleanField(default=False)
# Is the file a WebExtension?
is_webextension = models.BooleanField(default=False)
# Is the file a special "Mozilla Signed Extension"
# see https://wiki.mozilla.org/Add-ons/InternalSigning
is_mozilla_signed_extension = models.BooleanField(default=False)
# The user has disabled this file and this was its status.
# STATUS_NULL means the user didn't disable the File - i.e. Mozilla did.
original_status = models.PositiveSmallIntegerField(default=amo.STATUS_NULL)
class Meta(ModelBase.Meta):
db_table = 'files'
indexes = [
models.Index(fields=('created', 'version'), name='created_idx'),
models.Index(fields=('binary_components', ),
name='files_cedd2560'),
models.Index(fields=('datestatuschanged', 'version'),
name='statuschanged_idx'),
models.Index(fields=('status', ), name='status'),
]
def __str__(self):
return str(self.id)
@property
def has_been_validated(self):
try:
self.validation
except FileValidation.DoesNotExist:
return False
else:
return True
@property
def automated_signing(self):
"""True if this file is eligible for automated signing. This currently
means that either its version is unlisted."""
return self.version.channel == amo.RELEASE_CHANNEL_UNLISTED
def get_file_cdn_url(self, attachment=False):
"""Return the URL for the file corresponding to this instance
on the CDN."""
if attachment:
host = posixpath.join(user_media_url('addons'), '_attachments')
else:
host = user_media_url('addons')
return posixpath.join(
*map(force_bytes, [host, self.version.addon.id, self.filename]))
def get_url_path(self, attachment=False):
return self._make_download_url('downloads.file', attachment=attachment)
def _make_download_url(self, view_name, attachment=False):
kwargs = {'file_id': self.pk}
if attachment:
kwargs['type'] = 'attachment'
url = os.path.join(reverse(view_name, kwargs=kwargs), self.filename)
return url
@classmethod
def from_upload(cls, upload, version, parsed_data=None):
"""
Create a File instance from a FileUpload, a Version and the parsed_data
generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results."""
assert parsed_data is not None
file_ = cls(version=version)
upload_path = force_str(nfd_str(upload.path))
ext = force_str(os.path.splitext(upload_path)[1])
file_.filename = file_.generate_filename(extension=ext or '.xpi')
# Size in bytes.
file_.size = storage.size(upload_path)
file_.is_restart_required = parsed_data.get('is_restart_required',
False)
file_.strict_compatibility = parsed_data.get('strict_compatibility',
False)
file_.is_experiment = parsed_data.get('is_experiment', False)
file_.is_webextension = parsed_data.get('is_webextension', False)
file_.is_mozilla_signed_extension = parsed_data.get(
'is_mozilla_signed_extension', False)
file_.hash = file_.generate_hash(upload_path)
file_.original_hash = file_.hash
file_.save()
if file_.is_webextension:
permissions = list(parsed_data.get('permissions', []))
optional_permissions = list(
parsed_data.get('optional_permissions', []))
# devtools_page isn't in permissions block but treated as one
# if a custom devtools page is added by an addon
if 'devtools_page' in parsed_data:
permissions.append('devtools')
# Add content_scripts host matches too.
for script in parsed_data.get('content_scripts', []):
permissions.extend(script.get('matches', []))
if permissions or optional_permissions:
WebextPermission.objects.create(
permissions=permissions,
optional_permissions=optional_permissions,
file=file_,
)
log.info('New file: %r from %r' % (file_, upload))
# Move the uploaded file from the temp location.
copy_stored_file(upload_path, file_.current_file_path)
if upload.validation:
validation = json.loads(upload.validation)
FileValidation.from_json(file_, validation)
return file_
def generate_hash(self, filename=None):
"""Generate a hash for a file."""
with open(filename or self.current_file_path, 'rb') as fobj:
return 'sha256:{}'.format(get_sha256(fobj))
def generate_filename(self, extension=None):
"""
Files are in the format of:
{addon_name}-{version}-{apps}
(-{platform} for some of the old ones from back when we had multiple
platforms)
"""
parts = []
addon = self.version.addon
# slugify drops unicode so we may end up with an empty string.
# Apache did not like serving unicode filenames (bug 626587).
extension = extension or '.xpi'
name = slugify(addon.name).replace('-', '_') or 'addon'
parts.append(name)
parts.append(self.version.version)
if addon.type not in amo.NO_COMPAT and self.version.compatible_apps:
apps = '+'.join(
sorted([a.shortername for a in self.version.compatible_apps]))
parts.append(apps)
self.filename = '-'.join(parts) + extension
return self.filename
_pretty_filename = re.compile(r'(?P<slug>[a-z0-7_]+)(?P<suffix>.*)')
def pretty_filename(self, maxlen=20):
"""Displayable filename.
Truncates filename so that the slug part fits maxlen.
"""
m = self._pretty_filename.match(self.filename)
if not m:
return self.filename
if len(m.group('slug')) < maxlen:
return self.filename
return '%s...%s' % (m.group('slug')[0:(maxlen - 3)], m.group('suffix'))
def latest_xpi_url(self, attachment=False):
addon = self.version.addon
kw = {'addon_id': addon.slug}
if attachment:
kw['type'] = 'attachment'
return os.path.join(
reverse('downloads.latest', kwargs=kw),
'addon-%s-latest%s' % (addon.pk, self.extension),
)
@property
def file_path(self):
return os.path.join(user_media_path('addons'),
str(self.version.addon_id), self.filename)
@property
def addon(self):
return self.version.addon
@property
def guarded_file_path(self):
return os.path.join(user_media_path('guarded_addons'),
str(self.version.addon_id), self.filename)
@property
def current_file_path(self):
"""Returns the current path of the file, whether or not it is
guarded."""
file_disabled = self.status == amo.STATUS_DISABLED
addon_disabled = self.addon.is_disabled
if file_disabled or addon_disabled:
return self.guarded_file_path
else:
return self.file_path
@property
def fallback_file_path(self):
"""Fallback path in case the file was disabled/re-enabled and not yet
moved - sort of the opposite to current_file_path. This should only be
used for things like code search or git extraction where we really want
the file contents no matter what."""
return (self.file_path if self.current_file_path
== self.guarded_file_path else self.guarded_file_path)
@property
def extension(self):
return os.path.splitext(self.filename)[-1]
def move_file(self, source_path, destination_path, log_message):
"""Move a file from `source_path` to `destination_path` and delete the
source directory if it's empty once the file has been successfully
moved.
Meant to move files from/to the guarded file path as they are disabled
or re-enabled.
IOError and UnicodeEncodeError are caught and logged."""
log_message = force_str(log_message)
try:
if storage.exists(source_path):
source_parent_path = os.path.dirname(source_path)
log.info(
log_message.format(source=source_path,
destination=destination_path))
move_stored_file(source_path, destination_path)
# Now that the file has been deleted, remove the directory if
# it exists to prevent the main directory from growing too
# much (#11464)
remaining_dirs, remaining_files = storage.listdir(
source_parent_path)
if len(remaining_dirs) == len(remaining_files) == 0:
storage.delete(source_parent_path)
except (UnicodeEncodeError, IOError):
msg = 'Move Failure: {} {}'.format(source_path, destination_path)
log.exception(msg)
def hide_disabled_file(self):
"""Move a file from the public path to the guarded file path."""
if not self.filename:
return
src, dst = self.file_path, self.guarded_file_path
self.move_file(src, dst,
'Moving disabled file: {source} => {destination}')
def unhide_disabled_file(self):
"""Move a file from guarded file path to the public file path."""
if not self.filename:
return
src, dst = self.guarded_file_path, self.file_path
self.move_file(src, dst,
'Moving undisabled file: {source} => {destination}')
@cached_property
def permissions(self):
if not self.is_webextension:
return []
try:
# Filter out any errant non-strings included in the manifest JSON.
# Remove any duplicate permissions.
permissions = set()
permissions = [
p for p in self._webext_permissions.permissions
if isinstance(p, str)
and not (p in permissions or permissions.add(p))
]
return permissions
except WebextPermission.DoesNotExist:
return []
@cached_property
def optional_permissions(self):
if not self.is_webextension:
return []
try:
# Filter out any errant non-strings included in the manifest JSON.
# Remove any duplicate optional permissions.
permissions = set()
permissions = [
p for p in self._webext_permissions.optional_permissions
if isinstance(p, str)
and not (p in permissions or permissions.add(p))
]
return permissions
except WebextPermission.DoesNotExist:
return []
class Collection(ModelBase):
id = PositiveAutoField(primary_key=True)
uuid = models.UUIDField(blank=True, unique=True, null=True)
name = TranslatedField(require_locale=False)
slug = models.CharField(max_length=30, blank=True, null=True)
# description can (and sometimes does) contain html and other unsanitized
# content. It must be cleaned before display - NoURLsField just strips the
# URL without doing any escaping.
description = NoURLsField(require_locale=False)
default_locale = models.CharField(
max_length=10, default='en-US', db_column='defaultlocale'
)
listed = models.BooleanField(
default=True, help_text='Collections are either listed or private.'
)
addon_count = models.PositiveIntegerField(default=0, db_column='addonCount')
addons = models.ManyToManyField(
Addon, through='CollectionAddon', related_name='collections'
)
author = models.ForeignKey(
UserProfile, null=True, related_name='collections', on_delete=models.CASCADE
)
objects = CollectionManager()
class Meta(ModelBase.Meta):
db_table = 'collections'
indexes = [
models.Index(fields=('created',), name='collections_created_idx'),
models.Index(fields=('listed',), name='collections_listed_idx'),
models.Index(fields=('slug',), name='collections_slug_idx'),
]
constraints = [
models.UniqueConstraint(fields=('author', 'slug'), name='author_id'),
]
def __str__(self):
return f'{self.name} ({self.addon_count})'
def save(self, **kw):
if not self.uuid:
self.uuid = uuid.uuid4()
if not self.slug:
# Work with both, strings (if passed manually on .create()
# and UUID instances)
self.slug = str(self.uuid).replace('-', '')[:30]
self.clean_slug()
super().save(**kw)
def clean_slug(self):
if not self.author:
return
qs = self.author.collections.using('default')
slugs = {slug: id for slug, id in qs.values_list('slug', 'id')}
if self.slug in slugs and slugs[self.slug] != self.id:
for idx in range(len(slugs)):
new = f'{self.slug}-{idx + 1}'
if new not in slugs:
self.slug = new
return
def get_url_path(self):
return reverse('collections.detail', args=[self.author_id, self.slug])
@classmethod
def get_fallback(cls):
return cls._meta.get_field('default_locale')
def add_addon(self, addon):
CollectionAddon.objects.get_or_create(addon=addon, collection=self)
def remove_addon(self, addon):
CollectionAddon.objects.filter(addon=addon, collection=self).delete()
def owned_by(self, user):
return user.id == self.author_id
def is_public(self):
return self.listed
@staticmethod
def transformer(collections):
if not collections:
return
author_ids = {c.author_id for c in collections}
authors = {u.id: u for u in UserProfile.objects.filter(id__in=author_ids)}
for c in collections:
c.author = authors.get(c.author_id)
@staticmethod
def post_save(sender, instance, **kwargs):
from . import tasks
if kwargs.get('raw'):
return
tasks.collection_meta.delay(instance.id)
def check_ownership(
self, request, require_owner, require_author, ignore_disabled, admin
):
"""
Used by acl.check_ownership to see if request.user has permissions for
the collection.
"""
from olympia.access import acl
return acl.check_collection_ownership(request, self, require_owner)
class File(OnChangeMixin, ModelBase):
id = PositiveAutoField(primary_key=True)
STATUS_CHOICES = amo.STATUS_CHOICES_FILE
version = models.ForeignKey('versions.Version',
related_name='files',
on_delete=models.CASCADE)
platform = models.PositiveIntegerField(
choices=amo.SUPPORTED_PLATFORMS_CHOICES,
default=amo.PLATFORM_ALL.id,
db_column="platform_id")
filename = models.CharField(max_length=255, default='')
size = models.PositiveIntegerField(default=0) # In bytes.
hash = models.CharField(max_length=255, default='')
# The original hash of the file, before we sign it, or repackage it in
# any other way.
original_hash = models.CharField(max_length=255, default='')
jetpack_version = models.CharField(max_length=10, null=True, blank=True)
status = models.PositiveSmallIntegerField(
choices=STATUS_CHOICES.items(), default=amo.STATUS_AWAITING_REVIEW)
datestatuschanged = models.DateTimeField(null=True, auto_now_add=True)
is_restart_required = models.BooleanField(default=False)
strict_compatibility = models.BooleanField(default=False)
# The XPI contains JS that calls require("chrome").
requires_chrome = models.BooleanField(default=False)
reviewed = models.DateTimeField(null=True, blank=True)
# The `binary` field is used to store the flags from amo-validator when it
# finds files with binary extensions or files that may contain binary
# content.
binary = models.BooleanField(default=False)
# The `binary_components` field is used to store the flag from
# amo-validator when it finds "binary-components" in the chrome manifest
# file, used for default to compatible.
binary_components = models.BooleanField(default=False, db_index=True)
# Serial number of the certificate use for the signature.
cert_serial_num = models.TextField(blank=True)
# Is the file signed by Mozilla?
is_signed = models.BooleanField(default=False)
# Is the file a multi-package?
# https://developer.mozilla.org/en-US/docs/Multiple_Item_Packaging
is_multi_package = models.BooleanField(default=False)
# Is the file an experiment (see bug 1220097)?
is_experiment = models.BooleanField(default=False)
# Is the file a WebExtension?
is_webextension = models.BooleanField(default=False)
# Is the file a special "Mozilla Signed Extension"
# see https://wiki.mozilla.org/Add-ons/InternalSigning
is_mozilla_signed_extension = models.BooleanField(default=False)
# The user has disabled this file and this was its status.
# STATUS_NULL means the user didn't disable the File - i.e. Mozilla did.
original_status = models.PositiveSmallIntegerField(default=amo.STATUS_NULL)
class Meta(ModelBase.Meta):
db_table = 'files'
def __unicode__(self):
return unicode(self.id)
def get_platform_display(self):
return force_text(amo.PLATFORMS[self.platform].name)
@property
def has_been_validated(self):
try:
self.validation
except FileValidation.DoesNotExist:
return False
else:
return True
@property
def automated_signing(self):
"""True if this file is eligible for automated signing. This currently
means that either its version is unlisted."""
return self.version.channel == amo.RELEASE_CHANNEL_UNLISTED
def get_file_cdn_url(self, attachment=False):
"""Return the URL for the file corresponding to this instance
on the CDN."""
if attachment:
host = posixpath.join(user_media_url('addons'), '_attachments')
else:
host = user_media_url('addons')
return posixpath.join(
*map(force_bytes, [host, self.version.addon.id, self.filename]))
def get_url_path(self, src, attachment=False):
return self._make_download_url('downloads.file',
src,
attachment=attachment)
def _make_download_url(self, view_name, src, attachment=False):
kwargs = {'file_id': self.pk}
if attachment:
kwargs['type'] = 'attachment'
url = os.path.join(reverse(view_name, kwargs=kwargs), self.filename)
return absolutify(urlparams(url, src=src))
@classmethod
def from_upload(cls, upload, version, platform, parsed_data=None):
"""
Create a File instance from a FileUpload, a Version, a platform id
and the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results."""
assert parsed_data is not None
file_ = cls(version=version, platform=platform)
upload.path = force_bytes(nfd_str(upload.path))
ext = os.path.splitext(upload.path)[1]
if ext == '.jar':
ext = '.xpi'
file_.filename = file_.generate_filename(extension=ext or '.xpi')
# Size in bytes.
file_.size = storage.size(upload.path)
data = cls.get_jetpack_metadata(upload.path)
if 'sdkVersion' in data and data['sdkVersion']:
file_.jetpack_version = data['sdkVersion'][:10]
file_.is_restart_required = parsed_data.get('is_restart_required',
False)
file_.strict_compatibility = parsed_data.get('strict_compatibility',
False)
file_.is_multi_package = parsed_data.get('is_multi_package', False)
file_.is_experiment = parsed_data.get('is_experiment', False)
file_.is_webextension = parsed_data.get('is_webextension', False)
file_.is_mozilla_signed_extension = parsed_data.get(
'is_mozilla_signed_extension', False)
file_.hash = file_.generate_hash(upload.path)
file_.original_hash = file_.hash
if upload.validation:
validation = json.loads(upload.validation)
if validation['metadata'].get('requires_chrome'):
file_.requires_chrome = True
file_.save()
if file_.is_webextension:
permissions = list(parsed_data.get('permissions', []))
# Add content_scripts host matches too.
for script in parsed_data.get('content_scripts', []):
permissions.extend(script.get('matches', []))
if permissions:
WebextPermission.objects.create(permissions=permissions,
file=file_)
log.debug('New file: %r from %r' % (file_, upload))
# Move the uploaded file from the temp location.
copy_stored_file(upload.path, file_.current_file_path)
if upload.validation:
FileValidation.from_json(file_, validation)
return file_
@classmethod
def get_jetpack_metadata(cls, path):
data = {'sdkVersion': None}
try:
zip_ = zipfile.ZipFile(path)
except (zipfile.BadZipfile, IOError):
# This path is not an XPI. It's probably an app manifest.
return data
if 'package.json' in zip_.namelist():
data['sdkVersion'] = "jpm"
else:
name = 'harness-options.json'
if name in zip_.namelist():
try:
opts = json.load(zip_.open(name))
except ValueError as exc:
log.info('Could not parse harness-options.json in %r: %s' %
(path, exc))
else:
data['sdkVersion'] = opts.get('sdkVersion')
return data
def generate_hash(self, filename=None):
"""Generate a hash for a file."""
hash = hashlib.sha256()
with open(filename or self.current_file_path, 'rb') as obj:
for chunk in iter(lambda: obj.read(1024), ''):
hash.update(chunk)
return 'sha256:%s' % hash.hexdigest()
def generate_filename(self, extension=None):
"""
Files are in the format of:
{addon_name}-{version}-{apps}-{platform}
"""
parts = []
addon = self.version.addon
# slugify drops unicode so we may end up with an empty string.
# Apache did not like serving unicode filenames (bug 626587).
extension = extension or '.xpi'
name = slugify(addon.name).replace('-', '_') or 'addon'
parts.append(name)
parts.append(self.version.version)
if addon.type not in amo.NO_COMPAT and self.version.compatible_apps:
apps = '+'.join(
sorted([a.shortername for a in self.version.compatible_apps]))
parts.append(apps)
if self.platform and self.platform != amo.PLATFORM_ALL.id:
parts.append(amo.PLATFORMS[self.platform].shortname)
self.filename = '-'.join(parts) + extension
return self.filename
_pretty_filename = re.compile(r'(?P<slug>[a-z0-7_]+)(?P<suffix>.*)')
def pretty_filename(self, maxlen=20):
"""Displayable filename.
Truncates filename so that the slug part fits maxlen.
"""
m = self._pretty_filename.match(self.filename)
if not m:
return self.filename
if len(m.group('slug')) < maxlen:
return self.filename
return u'%s...%s' % (m.group('slug')[0:(maxlen - 3)],
m.group('suffix'))
def latest_xpi_url(self, attachment=False):
addon = self.version.addon
kw = {'addon_id': addon.slug}
if self.platform != amo.PLATFORM_ALL.id:
kw['platform'] = self.platform
if attachment:
kw['type'] = 'attachment'
return os.path.join(reverse('downloads.latest', kwargs=kw),
'addon-%s-latest%s' % (addon.pk, self.extension))
def eula_url(self):
return reverse('addons.eula', args=[self.version.addon_id, self.id])
@property
def file_path(self):
return os.path.join(user_media_path('addons'),
str(self.version.addon_id), self.filename)
@property
def addon(self):
return self.version.addon
@property
def guarded_file_path(self):
return os.path.join(user_media_path('guarded_addons'),
str(self.version.addon_id), self.filename)
@property
def current_file_path(self):
"""Returns the current path of the file, whether or not it is
guarded."""
file_disabled = self.status == amo.STATUS_DISABLED
addon_disabled = self.addon.is_disabled
if file_disabled or addon_disabled:
return self.guarded_file_path
else:
return self.file_path
@property
def extension(self):
return os.path.splitext(self.filename)[-1]
def move_file(self, source, destination, log_message):
"""Move a file from `source` to `destination`."""
# Make sure we are passing bytes to Python's io system.
source, destination = force_bytes(source), force_bytes(destination)
try:
if storage.exists(source):
log.info(
log_message.format(source=source, destination=destination))
move_stored_file(source, destination)
except (UnicodeEncodeError, IOError):
msg = 'Move Failure: {} {}'.format(force_bytes(source),
force_bytes(destination))
log.exception(msg)
def hide_disabled_file(self):
"""Move a disabled file to the guarded file path."""
if not self.filename:
return
src, dst = self.file_path, self.guarded_file_path
self.move_file(src, dst,
'Moving disabled file: {source} => {destination}')
def unhide_disabled_file(self):
if not self.filename:
return
src, dst = self.guarded_file_path, self.file_path
self.move_file(src, dst,
'Moving undisabled file: {source} => {destination}')
_get_localepicker = re.compile('^locale browser ([\w\-_]+) (.*)$', re.M)
@memoize(prefix='localepicker', timeout=None)
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
try:
zip = SafeZip(self.file_path)
except (zipfile.BadZipfile, IOError):
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' % (self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
@property
def webext_permissions(self):
"""Return permissions that should be displayed, with descriptions, in
defined order:
1) Either the match all permission, if present (e.g. <all-urls>), or
match urls for sites (<all-urls> takes preference over match urls)
2) nativeMessaging permission, if present
3) other known permissions in alphabetical order
"""
knowns = list(
WebextPermissionDescription.objects.filter(
name__in=self.webext_permissions_list))
urls = []
match_url = None
for name in self.webext_permissions_list:
if re.match(WebextPermissionDescription.MATCH_ALL_REGEX, name):
match_url = WebextPermissionDescription.ALL_URLS_PERMISSION
elif name == WebextPermission.NATIVE_MESSAGING_NAME:
# Move nativeMessaging to front of the list
for index, perm in enumerate(knowns):
if perm.name == WebextPermission.NATIVE_MESSAGING_NAME:
knowns.pop(index)
knowns.insert(0, perm)
break
elif '//' in name:
# Filter out match urls so we can group them.
urls.append(name)
# Other strings are unknown permissions we don't care about
if match_url is None and len(urls) == 1:
match_url = Permission(
u'single-match',
ugettext(u'Access your data for {name}').format(name=urls[0]))
elif match_url is None and len(urls) > 1:
details = (u'<details><summary>{copy}</summary><ul>{sites}</ul>'
u'</details>')
copy = ugettext(u'Access your data on the following websites:')
sites = ''.join(
[u'<li>%s</li>' % jinja2_escape(name) for name in urls])
match_url = Permission(
u'multiple-match',
mark_safe(details.format(copy=copy, sites=sites)))
return ([match_url] if match_url else []) + knowns
@cached_property
def webext_permissions_list(self):
if not self.is_webextension:
return []
try:
# Filter out any errant non-strings included in the manifest JSON.
# Remove any duplicate permissions.
permissions = set()
permissions = [
p for p in self._webext_permissions.permissions
if isinstance(p, basestring)
and not (p in permissions or permissions.add(p))
]
return permissions
except WebextPermission.DoesNotExist:
return []
