class Website(ModelBase):
# Identifier used for the initial e.me import.
moz_id = models.PositiveIntegerField(null=True, unique=True, blank=True)
# The default_locale used for translated fields. See get_fallback() method
# below.
default_locale = models.CharField(max_length=10,
default=settings.LANGUAGE_CODE)
# The Website URL.
url = models.URLField(max_length=255, blank=True, null=True)
# The Website mobile-specific URL, if one exists.
mobile_url = models.URLField(max_length=255, blank=True, null=True)
# The Website TV-specific URL, if one exists.
tv_url = models.URLField(max_length=255, blank=True, null=True)
# The <title> for the Website, used in search, not exposed to the frontend.
title = TranslatedField()
# The name and optionnal short name for the Website, used in the detail
# page and listing pages, respectively.
name = TranslatedField()
short_name = TranslatedField()
# Developer name (only used for TV)
developer_name = TranslatedField(null=True)
# Description.
description = TranslatedField()
# Website keywords.
keywords = models.ManyToManyField(Tag)
# Regions the website is known to be relevant in, used for search boosting.
# Stored as a JSON list of ids.
preferred_regions = JSONField(default=None)
# Categories, similar to apps. Stored as a JSON list of names.
categories = JSONField(default=None)
# Devices, similar to apps. Stored a JSON list of ids.
devices = JSONField(default=None)
# Icon content-type.
icon_type = models.CharField(max_length=25, blank=True)
# Icon cache-busting hash.
icon_hash = models.CharField(max_length=8, blank=True)
# Promo image cache-busting hash.
promo_img_hash = models.CharField(max_length=8, blank=True, null=True)
# Date & time the entry was last updated.
last_updated = models.DateTimeField(db_index=True, auto_now_add=True)
# Status, similar to apps. See WebsiteManager.valid() above.
status = models.PositiveIntegerField(choices=STATUS_CHOICES.items(),
default=STATUS_NULL)
# Whether the website entry is disabled (not shown in frontend, regardless
# of status) or not.
is_disabled = models.BooleanField(default=False)
tv_featured = models.PositiveIntegerField(null=True)
objects = WebsiteManager()
class Meta:
ordering = (('-last_updated'), )
@classmethod
def get_fallback(cls):
return cls._meta.get_field('default_locale')
@classmethod
def get_indexer(self):
return WebsiteIndexer
def __unicode__(self):
return unicode(self.url or '(no url set)')
@property
def device_names(self):
return [DEVICE_TYPES[device_id].api_name for device_id in self.devices]
@property
def device_types(self):
return [DEVICE_TYPES[device_id] for device_id in self.devices]
def is_dummy_content_for_qa(self):
"""
Returns whether this app is a dummy app used for testing only or not.
"""
# Change this when we start having dummy websites for QA purposes, see
# Webapp implementation.
return False
def get_icon_dir(self):
return os.path.join(settings.WEBSITE_ICONS_PATH, str(self.pk / 1000))
def get_icon_url(self, size):
icon_name = '{icon}-{{size}}.png'.format(
icon=DEFAULT_ICONS[self.pk % len(DEFAULT_ICONS)])
return get_icon_url(static_url('WEBSITE_ICON_URL'),
self,
size,
default_format=icon_name)
def get_promo_img_dir(self):
return os.path.join(settings.WEBSITE_PROMO_IMG_PATH,
str(self.id / 1000))
def get_promo_img_url(self, size):
if not self.promo_img_hash:
return ''
return get_promo_img_url(static_url('WEBSITE_PROMO_IMG_URL'),
self,
size,
default_format='website-promo-{size}.png')
def get_url_path(self):
return reverse('website.detail', kwargs={'pk': self.pk})
def get_preferred_regions(self, sort_by='slug'):
"""
Return a list of region objects the website is preferred in, e.g.::
[<class 'mkt.constants.regions.GBR'>, ...]
"""
_regions = map(mkt.regions.REGIONS_CHOICES_ID_DICT.get,
self.preferred_regions)
return sorted(_regions, key=operator.attrgetter(sort_by))
class Extension(ModelBase):
# Automatically handled fields.
deleted = models.BooleanField(default=False, editable=False)
icon_hash = models.CharField(max_length=8, blank=True)
last_updated = models.DateTimeField(blank=True, null=True, editable=False)
status = models.PositiveSmallIntegerField(choices=STATUS_CHOICES.items(),
default=STATUS_NULL,
editable=False)
uuid = UUIDField(auto=True, editable=False)
# Fields for which the manifest is the source of truth - can't be
# overridden by the API.
author = models.CharField(default='', editable=False, max_length=128)
default_language = models.CharField(default=settings.LANGUAGE_CODE,
editable=False,
max_length=10)
description = TranslatedField(default=None, editable=False)
name = TranslatedField(default=None, editable=False)
# Fields that can be modified using the API.
authors = models.ManyToManyField('users.UserProfile')
disabled = models.BooleanField(default=False)
slug = models.CharField(max_length=35, null=True, unique=True)
objects = ManagerBase.from_queryset(ExtensionQuerySet)()
manifest_is_source_of_truth_fields = ('author', 'description',
'default_language', 'name')
class Meta:
ordering = ('-id', )
index_together = (('deleted', 'disabled', 'status'), )
@cached_property(writable=True)
def latest_public_version(self):
return self.versions.without_deleted().public().latest('pk')
@cached_property(writable=True)
def latest_version(self):
return self.versions.without_deleted().latest('pk')
def block(self):
"""Block this Extension.
When in this state the Extension should not be editable by the
developers at all; not visible publicly; not searchable by users; but
should be shown in the developer's dashboard, as 'Blocked'."""
self.update(status=STATUS_BLOCKED)
def clean_slug(self):
return clean_slug(self, slug_field='slug')
def delete(self, *args, **kwargs):
"""Delete this instance.
By default, a soft-delete is performed, only hiding the instance from
the custom manager methods without actually removing it from the
database. pre_delete and post_delete signals are *not* sent in that
case. The slug will be set to None during the process.
Can be overridden by passing `hard_delete=True` keyword argument, in
which case it behaves like a regular delete() call instead."""
if self.is_blocked():
raise BlockedExtensionError
hard_delete = kwargs.pop('hard_delete', False)
if hard_delete:
# Real, hard delete.
return super(Extension, self).delete(*args, **kwargs)
# Soft delete.
# Since we have a unique constraint with slug, set it to None when
# deleting. Undelete should re-generate it - it might differ from the
# original slug, but that's why you should be careful when deleting...
self.update(deleted=True, slug=None)
@property
def devices(self):
"""Device ids the Extension is compatible with.
For now, hardcoded to only return Firefox OS."""
return [DEVICE_GAIA.id]
@property
def device_names(self):
"""Device names the Extension is compatible with.
Used by the API."""
return [DEVICE_TYPES[device_id].api_name for device_id in self.devices]
@classmethod
def extract_and_validate_upload(cls, upload):
"""Validate and extract manifest from a FileUpload instance.
Can raise ParseError."""
with private_storage.open(upload.path) as file_obj:
# The file will already have been uploaded at this point, so force
# the content type to make the ExtensionValidator happy. We just
# need to validate the contents.
file_obj.content_type = 'application/zip'
manifest_contents = ExtensionValidator(file_obj).validate()
return manifest_contents
@classmethod
def extract_manifest_fields(cls, manifest_data, fields=None):
"""Extract the specified `fields` from `manifest_data`, applying
transformations if necessary. If `fields` is absent, then use
`cls.manifest_is_source_of_truth_fields`."""
if fields is None:
fields = cls.manifest_is_source_of_truth_fields
data = {k: manifest_data[k] for k in fields if k in manifest_data}
# Determine default language to use for translations.
# Web Extensions Manifest contains locales (e.g. "en_US"), not
# languages (e.g. "en-US"). The field is also called differently as a
# result (default_locale vs default_language), so we need to transform
# both the key and the value before adding it to data. A default value
# needs to be set to correctly generate the translated fields below.
default_language = to_language(
manifest_data.get('default_locale',
cls._meta.get_field('default_language').default))
if 'default_language' in fields:
data['default_language'] = default_language
# Be nice and strip leading / trailing whitespace chars from
# strings.
for key, value in data.items():
if isinstance(value, basestring):
data[key] = value.strip()
# Translated fields should not be extracted as simple strings,
# otherwise we end up setting a locale on the translation that is
# dependent on the locale of the thread. Use dicts instead, always
# setting default_language as the language for now (since we don't
# support i18n in web extensions yet).
for field in cls._meta.translated_fields:
field_name = field.name
if field_name in data:
data[field_name] = {
default_language: manifest_data[field_name]
}
return data
@classmethod
def from_upload(cls, upload, user=None):
"""Handle creating/editing the Extension instance and saving it to db,
as well as file operations, from a FileUpload instance. Can throw
a ParseError or SigningError, so should always be called within a
try/except."""
manifest_contents = cls.extract_and_validate_upload(upload)
data = cls.extract_manifest_fields(manifest_contents)
# Check for name collision in the same locale for the same user.
default_language = data['default_language']
name = data['name'][default_language]
if user.extension_set.without_deleted().filter(
name__locale=default_language,
name__localized_string=name).exists():
raise ParseError(
_(u'An Add-on with the same name already exists in your '
u'submissions.'))
# Build a new instance.
instance = cls.objects.create(**data)
# Now that the instance has been saved, we can add the author and start
# saving version data. If everything checks out, a status will be set
# on the ExtensionVersion we're creating which will automatically be
# replicated on the Extension instance.
instance.authors.add(user)
version = ExtensionVersion.from_upload(
upload, parent=instance, manifest_contents=manifest_contents)
# Trigger icon fetch task asynchronously if necessary now that we have
# an extension and a version.
if 'icons' in manifest_contents:
fetch_icon.delay(instance.pk, version.pk)
return instance
@classmethod
def get_fallback(cls):
"""Class method returning the field holding the default language to use
in translations for this instance.
*Needs* to be called get_fallback() and *needs* to be a classmethod,
that's what the translations app requires."""
return cls._meta.get_field('default_language')
def get_icon_dir(self):
return os.path.join(settings.EXTENSION_ICONS_PATH, str(self.pk / 1000))
def get_icon_url(self, size):
return get_icon_url(static_url('EXTENSION_ICON_URL'), self, size)
@classmethod
def get_indexer(cls):
return ExtensionIndexer
def get_url_path(self):
return reverse('extension.detail', kwargs={'app_slug': self.slug})
@property
def icon_type(self):
return 'png' if self.icon_hash else ''
def is_blocked(self):
return self.status == STATUS_BLOCKED
def is_dummy_content_for_qa(self):
"""
Returns whether this extension is a dummy extension used for testing
only or not.
Used by mkt.search.utils.extract_popularity_trending_boost() - the
method needs to exist, but we are not using it yet.
"""
return False
def is_public(self):
return (not self.deleted and not self.disabled
and self.status == STATUS_PUBLIC)
@property
def mini_manifest(self):
"""Mini-manifest used for install/update on FxOS devices, in dict form.
It follows the Mozilla App Manifest format (because that's what FxOS
requires to install/update add-ons), *not* the Web Extension manifest
format.
"""
if self.is_blocked():
return {}
# Platform "translates" back the mini-manifest into an app manifest and
# verifies that some specific key properties in the real manifest match
# what's found in the mini-manifest. To prevent manifest mismatch
# errors, we need to copy those properties from the real manifest:
# name, description and author. To help Firefox OS display useful info
# to the user we also copy content_scripts and version.
# We don't bother with locales at the moment, this probably breaks
# extensions using https://developer.chrome.com/extensions/i18n but
# we'll deal with that later.
try:
version = self.latest_public_version
except ExtensionVersion.DoesNotExist:
return {}
manifest = version.manifest
mini_manifest = {
# 'id' here is the uuid, like in sign_file(). This is used by
# platform to do blocklisting.
'id': self.uuid,
'name': manifest['name'],
'package_path': version.download_url,
'size': version.size,
'version': manifest['version']
}
if 'author' in manifest:
# author is copied as a different key to match app manifest format.
mini_manifest['developer'] = {'name': manifest['author']}
if 'content_scripts' in manifest:
mini_manifest['content_scripts'] = manifest['content_scripts']
if 'description' in manifest:
mini_manifest['description'] = manifest['description']
return mini_manifest
@property
def mini_manifest_url(self):
return absolutify(
reverse('extension.mini_manifest', kwargs={'uuid': self.uuid}))
def save(self, *args, **kwargs):
if not self.deleted:
# Always clean slug before saving, to avoid clashes.
self.clean_slug()
return super(Extension, self).save(*args, **kwargs)
def __unicode__(self):
return u'%s: %s' % (self.pk, self.name)
def unblock(self):
"""Unblock this Extension. The original status is restored."""
self.status = STATUS_NULL
self.update_status_according_to_versions()
def undelete(self):
"""Undelete this instance, making it available to all manager methods
again and restoring its version number.
Return False if it was not marked as deleted, True otherwise.
Will re-generate a slug, that might differ from the original one if it
was taken in the meantime."""
if not self.deleted:
return False
self.clean_slug()
self.update(deleted=False, slug=self.slug)
return True
def update_manifest_fields_from_latest_public_version(self):
"""Update all fields for which the manifest is the source of truth
with the manifest from the latest public add-on."""
if self.is_blocked():
raise BlockedExtensionError
try:
version = self.latest_public_version
except ExtensionVersion.DoesNotExist:
return
if not version.manifest:
return
# Trigger icon fetch task asynchronously if necessary now that we have
# an extension and a version.
if 'icons' in version.manifest:
fetch_icon.delay(self.pk, version.pk)
# We need to re-extract the fields from manifest contents because some
# fields like default_language are transformed before being stored.
data = self.extract_manifest_fields(version.manifest)
return self.update(**data)
def update_status_according_to_versions(self):
"""Update `status`, `latest_version` and `latest_public_version`
properties depending on the `status` on the ExtensionVersion
instances attached to this Extension."""
if self.is_blocked():
raise BlockedExtensionError
# If there is a public version available, the extension should be
# public. If not, and if there is a pending version available, it
# should be pending. If not, and if there is a rejected version
# available, it should be rejected. Otherwise it should just be
# incomplete.
versions = self.versions.without_deleted()
if versions.public().exists():
self.update(status=STATUS_PUBLIC)
elif versions.pending().exists():
self.update(status=STATUS_PENDING)
elif versions.rejected().exists():
self.update(status=STATUS_REJECTED)
else:
self.update(status=STATUS_NULL)
# Delete latest_version and latest_public_version properties, since
# they are writable cached_properties they will be reset the next time
# they are accessed.
try:
if self.latest_version:
del self.latest_version
except ExtensionVersion.DoesNotExist:
pass
try:
if self.latest_public_version:
del self.latest_public_version
except ExtensionVersion.DoesNotExist:
pass
class Extension(ModelBase):
# Fields for which the manifest is the source of truth - can't be
# overridden by the API.
default_language = models.CharField(default=settings.LANGUAGE_CODE,
max_length=10)
manifest = JSONField()
version = models.CharField(max_length=255, default='')
# Fields that can be modified using the API.
authors = models.ManyToManyField('users.UserProfile')
name = TranslatedField(default=None)
slug = models.CharField(max_length=35, unique=True)
status = models.PositiveSmallIntegerField(choices=STATUS_CHOICES.items(),
db_index=True,
default=STATUS_NULL)
def clean_slug(self):
return clean_slug(self, slug_field='slug')
@property
def download_url(self):
raise NotImplementedError
@property
def filename(self):
return 'extension-%s.zip' % self.version
@property
def file_path(self):
return os.path.join(self.path_prefix, nfd_str(self.filename))
@classmethod
def from_upload(cls, upload, instance=None):
"""Handle creating/editing the Extension instance and saving it to db,
as well as file operations, from a FileUpload instance. Can throw
a ValidationError or SigningError, so should always be called within a
try/except."""
if instance is not None:
# Not implemented yet. Need to deal with versions correctly, we
# don't know yet if we want to keep older versions around or not,
# how status changes etc.
raise NotImplementedError
parser = ExtensionParser(upload, instance=instance)
data = parser.parse()
fields = ('version', 'name', 'default_language')
default_locale = data.get('default_locale')
if default_locale:
# We actually need language (en-US) for translations, not locale
# (en_US). The extension contains locales though, so transform the
# field in the manifest before storing in db.
data['default_language'] = to_language(default_locale)
# Filter out parsing results to only keep fields we store in db.
data = dict((k, v) for k, v in data.items() if k in fields)
# Build a new instance.
instance = cls(**data)
instance.manifest = parser.manifest_contents
instance.clean_slug()
instance.save()
# Now that the instance has been saved, we can generate a file path,
# move the file and set it to PENDING.
instance.handle_file_operations(upload)
instance.update(status=STATUS_PENDING)
return instance
@classmethod
def get_fallback(cls):
# Class method needed by the translations app.
return cls._meta.get_field('default_language')
def get_minifest_contents(self, force=False):
raise NotImplementedError
def get_package_path(self):
raise NotImplementedError
def handle_file_operations(self, upload):
"""Copy the file attached to a FileUpload to the Extension instance."""
upload.path = smart_path(nfd_str(upload.path))
if not self.slug:
raise RuntimeError(
'Trying to upload a file belonging to a slugless extension')
if private_storage.exists(self.file_path):
# The filename should not exist. If it does, it means we are trying
# to re-upload the same version. This should have been caught
# before, so just raise an exception.
raise RuntimeError(
'Trying to upload a file to a destination that already exists')
# Copy file from fileupload. This uses private_storage for now as the
# unreviewed, unsigned filename is private.
copy_stored_file(upload.path, self.file_path)
def is_public(self):
return self.status == STATUS_PUBLIC
@property
def manifest_url(self):
raise NotImplementedError
@property
def path_prefix(self):
return os.path.join(settings.ADDONS_PATH, 'extensions', str(self.pk))
def __unicode__(self):
return u'%s: %s' % (self.pk, self.name)