class L10nSettings(ModelBase): """Per-locale L10n Dashboard settings""" locale = models.CharField(max_length=30, default='', unique=True) motd = PurifiedField() team_homepage = models.CharField(max_length=255, default='', null=True) class Meta: db_table = 'l10n_settings'
class Version(OnChangeMixin, ModelBase): addon = models.ForeignKey('addons.Addon', related_name='versions', on_delete=models.CASCADE) license = models.ForeignKey('License', null=True) releasenotes = PurifiedField() approvalnotes = models.TextField(default='', null=True) version = models.CharField(max_length=255, default='0.1') version_int = models.BigIntegerField(null=True, editable=False) nomination = models.DateTimeField(null=True) reviewed = models.DateTimeField(null=True) has_info_request = models.BooleanField(default=False) has_editor_comment = models.BooleanField(default=False) deleted = models.BooleanField(default=False) source = models.FileField(upload_to=source_upload_path, null=True, blank=True) channel = models.IntegerField(choices=amo.RELEASE_CHANNEL_CHOICES, default=amo.RELEASE_CHANNEL_LISTED) # The order of those managers is very important: please read the lengthy # comment above the Addon managers declaration/instantiation. unfiltered = VersionManager(include_deleted=True) objects = VersionManager() class Meta(ModelBase.Meta): db_table = 'versions' ordering = ['-created', '-modified'] def __init__(self, *args, **kwargs): super(Version, self).__init__(*args, **kwargs) self.__dict__.update(version_dict(self.version or '')) def __unicode__(self): return jinja2.escape(self.version) def save(self, *args, **kw): if not self.version_int and self.version: v_int = version_int(self.version) # Magic number warning, this is the maximum size # of a big int in MySQL to prevent version_int overflow, for # people who have rather crazy version numbers. # http://dev.mysql.com/doc/refman/5.5/en/numeric-types.html if v_int < 9223372036854775807: self.version_int = v_int else: log.error('No version_int written for version %s, %s' % (self.pk, self.version)) super(Version, self).save(*args, **kw) return self @classmethod def from_upload(cls, upload, addon, platforms, channel, send_signal=True, source=None, is_beta=False, parsed_data=None): from olympia.addons.models import AddonFeatureCompatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') if parsed_data is None: parsed_data = utils.parse_addon(upload, addon) license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id version = cls.objects.create( addon=addon, version=parsed_data['version'], license_id=license_id, source=source, channel=channel, ) log.info('New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) # Update the add-on e10s compatibility since we're creating a new # version that may change that. e10s_compatibility = parsed_data.get('e10s_compatibility') if e10s_compatibility is not None: feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create( addon=addon)[0]) feature_compatibility.update(e10s=e10s_compatibility) compatible_apps = {} for app in parsed_data.get('apps', []): compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version.compatible_apps = compatible_apps if addon.type == amo.ADDON_SEARCH: # Search extensions are always for all platforms. platforms = [amo.PLATFORM_ALL.id] else: platforms = cls._make_safe_platform_files(platforms) for platform in platforms: File.from_upload(upload, version, platform, parsed_data=parsed_data, is_beta=is_beta) version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) if send_signal: version_uploaded.send(sender=version) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version @classmethod def _make_safe_platform_files(cls, platforms): """Make file platform translations until all download pages support desktop ALL + mobile ALL. See bug 646268. Returns platforms ids. """ pl_set = set(platforms) if pl_set == set((amo.PLATFORM_ALL.id, )): # Make it really ALL: return [amo.PLATFORM_ALL.id] has_mobile = amo.PLATFORM_ANDROID in pl_set has_desktop = any(p in amo.DESKTOP_PLATFORMS for p in pl_set) has_all = amo.PLATFORM_ALL in pl_set is_mixed = has_mobile and has_desktop if (is_mixed and has_all) or has_mobile: # Mixing desktop and mobile w/ ALL is not safe; # we have to split the files into exact platforms. new_plats = [] for platform in platforms: if platform == amo.PLATFORM_ALL.id: plats = amo.DESKTOP_PLATFORMS.keys() plats.remove(amo.PLATFORM_ALL.id) new_plats.extend(plats) else: new_plats.append(platform) return new_plats # Platforms are safe as is return platforms @property def path_prefix(self): return os.path.join(user_media_path('addons'), str(self.addon_id)) def license_url(self, impala=False): return reverse('addons.license', args=[self.addon.slug, self.version]) def get_url_path(self): if self.channel == amo.RELEASE_CHANNEL_UNLISTED: return '' return reverse('addons.versions', args=[self.addon.slug, self.version]) def delete(self, hard=False): log.info(u'Version deleted: %r (%s)' % (self, self.id)) activity.log_create(amo.LOG.DELETE_VERSION, self.addon, str(self.version)) if hard: super(Version, self).delete() else: # By default we soft delete so we can keep the files for comparison # and a record of the version number. self.files.update(status=amo.STATUS_DISABLED) self.deleted = True self.save() @property def is_user_disabled(self): return self.files.filter(status=amo.STATUS_DISABLED).exclude( original_status=amo.STATUS_NULL).exists() @is_user_disabled.setter def is_user_disabled(self, disable): # User wants to disable (and the File isn't already). if disable: for file in self.files.exclude(status=amo.STATUS_DISABLED).all(): file.update(original_status=file.status, status=amo.STATUS_DISABLED) # User wants to re-enable (and user did the disable, not Mozilla). else: for file in self.files.exclude( original_status=amo.STATUS_NULL).all(): file.update(status=file.original_status, original_status=amo.STATUS_NULL) @property def current_queue(self): """Return the current queue, or None if not in a queue.""" from olympia.editors.models import (ViewFullReviewQueue, ViewPendingQueue) if self.channel == amo.RELEASE_CHANNEL_UNLISTED: # Unlisted add-ons and their updates are automatically approved so # they don't get a queue. # TODO: when we've finished with unlisted/listed versions the # status of an all-unlisted addon will be STATUS_NULL so we won't # need this check. return None if self.addon.status == amo.STATUS_NOMINATED: return ViewFullReviewQueue elif self.addon.status == amo.STATUS_PUBLIC: return ViewPendingQueue return None @cached_property def all_activity(self): from olympia.activity.models import VersionLog # yucky al = (VersionLog.objects.filter( version=self.id).order_by('created').select_related( 'activity_log', 'version').no_cache()) return al @cached_property def compatible_apps(self): """Get a mapping of {APP: ApplicationVersion}.""" avs = self.apps.select_related('version') return self._compat_map(avs) @cached_property def compatible_apps_ordered(self): apps = self.compatible_apps.items() return sorted(apps, key=lambda v: v[0].short) def compatible_platforms(self): """Returns a dict of compatible file platforms for this version. The result is based on which app(s) the version targets. """ app_ids = [a.application for a in self.apps.all()] targets_mobile = amo.ANDROID.id in app_ids targets_other = any((id_ != amo.ANDROID.id) for id_ in app_ids) all_plats = {} if targets_other: all_plats.update(amo.DESKTOP_PLATFORMS) if targets_mobile: all_plats.update(amo.MOBILE_PLATFORMS) return all_plats @cached_property def is_compatible(self): """Returns tuple of compatibility and reasons why if not. Server side conditions for determining compatibility are: * The add-on is an extension (not a theme, app, etc.) * Has not opted in to strict compatibility. * Does not use binary_components in chrome.manifest. Note: The lowest maxVersion compat check needs to be checked separately. Note: This does not take into account the client conditions. """ compat = True reasons = [] if self.addon.type != amo.ADDON_EXTENSION: compat = False # TODO: We may want this. For now we think it may be confusing. # reasons.append(ugettext('Add-on is not an extension.')) if self.files.filter(binary_components=True).exists(): compat = False reasons.append(ugettext('Add-on uses binary components.')) if self.files.filter(strict_compatibility=True).exists(): compat = False reasons.append( ugettext( 'Add-on has opted into strict compatibility checking.')) return (compat, reasons) def is_compatible_app(self, app): """Returns True if the provided app passes compatibility conditions.""" appversion = self.compatible_apps.get(app) if appversion and app.id in amo.D2C_MAX_VERSIONS: return (version_int(appversion.max.version) >= version_int( amo.D2C_MAX_VERSIONS.get(app.id, '*'))) return False def compat_override_app_versions(self): """Returns the incompatible app versions range(s). If not ranges, returns empty list. Otherwise, this will return all the app version ranges that this particular version is incompatible with. """ from olympia.addons.models import CompatOverride cos = CompatOverride.objects.filter(addon=self.addon) if not cos: return [] app_versions = [] for co in cos: for range in co.collapsed_ranges(): if (version_int(range.min) <= version_int(self.version) <= version_int(range.max)): app_versions.extend([(a.min, a.max) for a in range.apps]) return app_versions @cached_property def all_files(self): """Shortcut for list(self.files.all()). Heavily cached.""" return list(self.files.all()) @cached_property def supported_platforms(self): """Get a list of supported platform names.""" return list(set(amo.PLATFORMS[f.platform] for f in self.all_files)) @property def status(self): return [ f.STATUS_CHOICES.get(f.status, ugettext('[status:%s]') % f.status) for f in self.all_files ] @property def statuses(self): """Unadulterated statuses, good for an API.""" return [(f.id, f.status) for f in self.all_files] def is_allowed_upload(self): """ Check that a file can be uploaded based on the files per platform for that type of addon. """ num_files = len(self.all_files) if self.addon.type == amo.ADDON_SEARCH: return num_files == 0 elif num_files == 0: return True elif amo.PLATFORM_ALL in self.supported_platforms: return False # We don't want new files once a review has been done. elif (not self.is_all_unreviewed and not self.is_beta and self.channel == amo.RELEASE_CHANNEL_LISTED): return False else: compatible = (v for k, v in self.compatible_platforms().items() if k != amo.PLATFORM_ALL.id) return bool(set(compatible) - set(self.supported_platforms)) def is_public(self): # To be public, a version must not be deleted, must belong to a public # addon, and all its attached files must have public status. try: return (not self.deleted and self.addon.is_public() and all(f.status == amo.STATUS_PUBLIC for f in self.all_files)) except ObjectDoesNotExist: return False @property def requires_restart(self): return any(file_.requires_restart for file_ in self.all_files) @property def is_webextension(self): return any(file_.is_webextension for file_ in self.all_files) @property def has_files(self): return bool(self.all_files) @property def is_unreviewed(self): return filter(lambda f: f.status in amo.UNREVIEWED_FILE_STATUSES, self.all_files) @property def is_all_unreviewed(self): return not bool([ f for f in self.all_files if f.status not in amo.UNREVIEWED_FILE_STATUSES ]) @property def is_beta(self): return any(f for f in self.all_files if f.status == amo.STATUS_BETA) @property def is_jetpack(self): return all(f.jetpack_version for f in self.all_files) @property def sources_provided(self): return bool(self.source) @property def admin_review(self): return self.addon.admin_review @classmethod def _compat_map(cls, avs): apps = {} for av in avs: app_id = av.application if app_id in amo.APP_IDS: apps[amo.APP_IDS[app_id]] = av return apps @classmethod def transformer(cls, versions): """Attach all the compatible apps and files to the versions.""" ids = set(v.id for v in versions) if not versions: return # FIXME: find out why we have no_cache() here and try to remove it. avs = (ApplicationsVersions.objects.filter( version__in=ids).select_related('min', 'max').no_cache()) files = File.objects.filter(version__in=ids).no_cache() def rollup(xs): groups = sorted_groupby(xs, 'version_id') return dict((k, list(vs)) for k, vs in groups) av_dict, file_dict = rollup(avs), rollup(files) for version in versions: v_id = version.id version.compatible_apps = cls._compat_map(av_dict.get(v_id, [])) version.all_files = file_dict.get(v_id, []) for f in version.all_files: f.version = version @classmethod def transformer_activity(cls, versions): """Attach all the activity to the versions.""" from olympia.activity.models import VersionLog # yucky ids = set(v.id for v in versions) if not versions: return al = (VersionLog.objects.filter( version__in=ids).order_by('created').select_related( 'activity_log', 'version').no_cache()) def rollup(xs): groups = sorted_groupby(xs, 'version_id') return dict((k, list(vs)) for k, vs in groups) al_dict = rollup(al) for version in versions: v_id = version.id version.all_activity = al_dict.get(v_id, []) def disable_old_files(self): """ Disable files from versions older than the current one and awaiting review. Used when uploading a new version. Does nothing if the current instance is unlisted or has beta files. """ if (self.channel == amo.RELEASE_CHANNEL_LISTED and not self.files.filter(status=amo.STATUS_BETA).exists()): qs = File.objects.filter( version__addon=self.addon_id, version__lt=self.id, version__deleted=False, status__in=[amo.STATUS_AWAITING_REVIEW, amo.STATUS_PENDING]) # Use File.update so signals are triggered. for f in qs: f.update(status=amo.STATUS_DISABLED) def reset_nomination_time(self, nomination=None): if not self.nomination or nomination: nomination = nomination or datetime.datetime.now() # We need signal=False not to call update_status (which calls us). self.update(nomination=nomination, _signal=False) # But we need the cache to be flushed. Version.objects.invalidate(self) def inherit_nomination(self, from_statuses=None): last_ver = (Version.objects.filter( addon=self.addon, channel=amo.RELEASE_CHANNEL_LISTED).exclude( nomination=None).exclude(id=self.pk).order_by('-nomination')) if from_statuses: last_ver = last_ver.filter(files__status__in=from_statuses) if last_ver.exists(): self.reset_nomination_time(nomination=last_ver[0].nomination) @property def unreviewed_files(self): """A File is unreviewed if its status is amo.STATUS_AWAITING_REVIEW.""" return self.files.filter(status=amo.STATUS_AWAITING_REVIEW) @property def is_ready_for_auto_approval(self): """Return whether or not this version could be *considered* for auto-approval. Does not necessarily mean that it would be auto-approved, just that it passes the most basic criteria to be considered a candidate by the auto_approve command.""" return (self.addon.status == amo.STATUS_PUBLIC and self.addon.type == amo.ADDON_EXTENSION and self.is_webextension and self.is_unreviewed and self.channel == amo.RELEASE_CHANNEL_LISTED) @property def was_auto_approved(self): """Return whether or not this version was auto-approved.""" from olympia.editors.models import AutoApprovalSummary try: return (self.is_public() and self.autoapprovalsummary.verdict == amo.AUTO_APPROVED) except AutoApprovalSummary.DoesNotExist: pass return False
class Version(OnChangeMixin, ModelBase): id = PositiveAutoField(primary_key=True) addon = models.ForeignKey('addons.Addon', related_name='versions', on_delete=models.CASCADE) license = models.ForeignKey('License', null=True, blank=True, on_delete=models.SET_NULL) release_notes = PurifiedField(db_column='releasenotes', short=False) approval_notes = models.TextField(db_column='approvalnotes', default='', null=True, blank=True) version = VersionStringField(max_length=255, default='0.1') nomination = models.DateTimeField(null=True) reviewed = models.DateTimeField(null=True) deleted = models.BooleanField(default=False) source = models.FileField(upload_to=source_upload_path, null=True, blank=True, max_length=255) channel = models.IntegerField(choices=amo.RELEASE_CHANNEL_CHOICES, default=amo.RELEASE_CHANNEL_LISTED) git_hash = models.CharField(max_length=40, blank=True) needs_human_review = models.BooleanField(default=False) # The order of those managers is very important: please read the lengthy # comment above the Addon managers declaration/instantiation. unfiltered = VersionManager(include_deleted=True) objects = VersionManager() # See UnfilteredVersionManagerForRelations() docstring for usage of this # special manager. unfiltered_for_relations = UnfilteredVersionManagerForRelations() class Meta(ModelBase.Meta): db_table = 'versions' # This is very important: please read the lengthy comment in Addon.Meta # description base_manager_name = 'unfiltered' ordering = ['-created', '-modified'] indexes = [ models.Index(fields=('addon', ), name='addon_id'), models.Index(fields=('license', ), name='license_id'), ] constraints = [ models.UniqueConstraint( fields=('addon', 'version'), name='versions_addon_id_version_5a2e75b6_uniq', ), ] def __str__(self): return markupsafe.escape(self.version) @classmethod def from_upload(cls, upload, addon, selected_apps, channel, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of compatible app ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ from olympia.addons.models import AddonReviewerFlags from olympia.addons.utils import RestrictionChecker from olympia.git.utils import create_git_extraction_entry assert parsed_data is not None if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') if upload.addon and upload.addon != addon: raise VersionCreateError( 'FileUpload was made for a different Addon') if not upload.user or not upload.ip_address or not upload.source: raise VersionCreateError( 'FileUpload does not have some required fields') if not upload.user.last_login_ip or not upload.user.email: raise VersionCreateError( 'FileUpload user does not have some required fields') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id approval_notes = None if parsed_data.get('is_mozilla_signed_extension'): approval_notes = ( 'This version has been signed with Mozilla internal certificate.' ) version = cls.objects.create( addon=addon, approval_notes=approval_notes, version=parsed_data['version'], license_id=license_id, channel=channel, ) email = upload.user.email if upload.user and upload.user.email else '' with core.override_remote_addr(upload.ip_address): # The following log statement is used by foxsec-pipeline. # We override the IP because it might be called from a task and we # want the original IP from the submitter. log.info( f'New version: {version!r} ({version.id}) from {upload!r}', extra={ 'email': email, 'guid': addon.guid, 'upload': upload.uuid.hex, 'user_id': upload.user_id, 'from_api': upload.source == amo.UPLOAD_SOURCE_API, }, ) activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=upload.user or get_task_user()) if addon.type == amo.ADDON_STATICTHEME: # We don't let developers select apps for static themes selected_apps = [app.id for app in amo.APP_USAGE] compatible_apps = {} for app in parsed_data.get('apps', []): if app.id not in selected_apps: # If the user chose to explicitly deselect Firefox for Android # we're not creating the respective `ApplicationsVersions` # which will have this add-on then be listed only for # Firefox specifically. continue compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # Pre-generate _compatible_apps property to avoid accidentally # triggering queries with that instance later. version._compatible_apps = compatible_apps # Create relevant file and update the all_files cached property on the # Version, because we might need it afterwards. version.all_files = [ File.from_upload( upload=upload, version=version, parsed_data=parsed_data, ) ] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to its permanent location, delete it # from storage. Keep the FileUpload instance (it gets cleaned up by a # cron eventually some time after its creation, in amo.cron.gc()), # making sure it's associated with the add-on instance. storage.delete(upload.path) upload.path = '' if upload.addon is None: upload.addon = addon upload.save() version_uploaded.send(instance=version, sender=Version) if version.is_webextension: if (waffle.switch_is_active('enable-yara') or waffle.switch_is_active('enable-customs') or waffle.switch_is_active('enable-wat')): ScannerResult.objects.filter(upload_id=upload.id).update( version=version) if waffle.switch_is_active('enable-uploads-commit-to-git-storage'): # Schedule this version for git extraction. transaction.on_commit( lambda: create_git_extraction_entry(version=version)) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): theme_data = parsed_data.get('theme', {}) generate_static_theme_preview(theme_data, version.pk) # Reset add-on reviewer flags to disable auto-approval and require # admin code review if the package has already been signed by mozilla. reviewer_flags_defaults = {} is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension') if upload.validation_timeout: reviewer_flags_defaults['needs_admin_code_review'] = True if is_mozilla_signed and addon.type != amo.ADDON_LPAPP: reviewer_flags_defaults['needs_admin_code_review'] = True reviewer_flags_defaults['auto_approval_disabled'] = True # Check if the approval should be restricted if not RestrictionChecker(upload=upload).is_auto_approval_allowed(): flag = ('auto_approval_disabled' if channel == amo.RELEASE_CHANNEL_LISTED else 'auto_approval_disabled_unlisted') reviewer_flags_defaults[flag] = True if reviewer_flags_defaults: AddonReviewerFlags.objects.update_or_create( addon=addon, defaults=reviewer_flags_defaults) # Authors need to be notified about auto-approval delay again since # they are submitting a new version. addon.reset_notified_about_auto_approval_delay() # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version def license_url(self, impala=False): return reverse('addons.license', args=[self.addon.slug, self.version]) def get_url_path(self): if self.channel == amo.RELEASE_CHANNEL_UNLISTED: return '' return reverse('addons.versions', args=[self.addon.slug]) def delete(self, hard=False): # To avoid a circular import from .tasks import delete_preview_files log.info(f'Version deleted: {self!r} ({self.id})') activity.log_create(amo.LOG.DELETE_VERSION, self.addon, str(self.version)) if hard: super().delete() else: # By default we soft delete so we can keep the files for comparison # and a record of the version number. self.files.update(status=amo.STATUS_DISABLED) self.deleted = True self.save() # Clear pending rejection flag (we have the activity log for # records purposes, the flag serves no purpose anymore if the # version is deleted). VersionReviewerFlags.objects.filter(version=self).update( pending_rejection=None) previews_pks = list( VersionPreview.objects.filter(version__id=self.id).values_list( 'id', flat=True)) for preview_pk in previews_pks: delete_preview_files.delay(preview_pk) @property def is_user_disabled(self): return (self.files.filter(status=amo.STATUS_DISABLED).exclude( original_status=amo.STATUS_NULL).exists()) @is_user_disabled.setter def is_user_disabled(self, disable): # User wants to disable (and the File isn't already). if disable: activity.log_create(amo.LOG.DISABLE_VERSION, self.addon, self) for file in self.files.exclude(status=amo.STATUS_DISABLED).all(): file.update(original_status=file.status, status=amo.STATUS_DISABLED) # User wants to re-enable (and user did the disable, not Mozilla). else: activity.log_create(amo.LOG.ENABLE_VERSION, self.addon, self) for file in self.files.exclude( original_status=amo.STATUS_NULL).all(): file.update(status=file.original_status, original_status=amo.STATUS_NULL) @cached_property def all_activity(self): # prefetch_related() and not select_related() the ActivityLog to make # sure its transformer is called. return self.versionlog_set.prefetch_related('activity_log').order_by( 'created') @property def compatible_apps(self): # Dicts and search providers don't have compatibility info. # Fake one for them. if self.addon and self.addon.type in amo.NO_COMPAT: return {app: None for app in amo.APP_TYPE_SUPPORT[self.addon.type]} # Otherwise, return _compatible_apps which is a cached property that # is filled by the transformer, or simply calculated from the related # compat instances. return self._compatible_apps @cached_property def _compatible_apps(self): """Get a mapping of {APP: ApplicationsVersions}.""" return self._compat_map(self.apps.all().select_related('min', 'max')) @cached_property def compatible_apps_ordered(self): apps = self.compatible_apps.items() return sorted(apps, key=lambda v: v[0].short) @cached_property def is_compatible_by_default(self): """Returns whether or not the add-on is considered compatible by default.""" # Use self.all_files directly since that's cached and more potentially # prefetched through a transformer already return not any([ file for file in self.all_files if file.binary_components or file.strict_compatibility ]) def is_compatible_app(self, app): """Returns True if the provided app passes compatibility conditions.""" if self.addon.type in amo.NO_COMPAT: return True appversion = self.compatible_apps.get(app) if appversion and app.id in amo.D2C_MIN_VERSIONS: return version_int(appversion.max.version) >= version_int( amo.D2C_MIN_VERSIONS.get(app.id, '*')) return False def compat_override_app_versions(self): """Returns the incompatible app versions range(s). If not ranges, returns empty list. Otherwise, this will return all the app version ranges that this particular version is incompatible with. """ overrides = list(self.addon.compatoverride_set.all()) if not overrides: return [] app_versions = [] for co in overrides: for range in co.collapsed_ranges(): if (version_int(range.min) <= version_int(self.version) <= version_int(range.max)): app_versions.extend([(a.min, a.max) for a in range.apps]) return app_versions @cached_property def all_files(self): """Shortcut for list(self.files.all()). Cached.""" return list(self.files.all()) @property def current_file(self): """Shortcut for selecting the first file from self.all_files""" return self.all_files[0] @property def status(self): return [ f.STATUS_CHOICES.get(f.status, gettext('[status:%s]') % f.status) for f in self.all_files ] @property def statuses(self): """Unadulterated statuses, good for an API.""" return [(f.id, f.status) for f in self.all_files] def is_public(self): # To be public, a version must not be deleted, must belong to a public # addon, and all its attached files must have public status. try: return (not self.deleted and self.addon.is_public() and all(f.status == amo.STATUS_APPROVED for f in self.all_files)) except ObjectDoesNotExist: return False @property def is_webextension(self): return any(file_.is_webextension for file_ in self.all_files) @property def is_mozilla_signed(self): """Is the file a special "Mozilla Signed Extension" See https://wiki.mozilla.org/Add-ons/InternalSigning for more details. We use that information to workaround compatibility limits for legacy add-ons and to avoid them receiving negative boosts compared to WebExtensions. See https://github.com/mozilla/addons-server/issues/6424 """ return all(file_.is_mozilla_signed_extension for file_ in self.all_files) @property def has_files(self): return bool(self.all_files) @property def is_unreviewed(self): return bool( list( filter(lambda f: f.status in amo.UNREVIEWED_FILE_STATUSES, self.all_files))) @property def is_all_unreviewed(self): return not bool([ f for f in self.all_files if f.status not in amo.UNREVIEWED_FILE_STATUSES ]) @property def sources_provided(self): return bool(self.source) def _compat_map(self, avs): apps = {} for av in avs: av.version = self app_id = av.application if app_id in amo.APP_IDS: apps[amo.APP_IDS[app_id]] = av return apps @classmethod def transformer(cls, versions): """Attach all the compatible apps and files to the versions.""" if not versions: return ids = {v.id for v in versions} avs = ApplicationsVersions.objects.filter( version__in=ids).select_related('min', 'max') files = File.objects.filter(version__in=ids) def rollup(xs): groups = sorted_groupby(xs, 'version_id') return {k: list(vs) for k, vs in groups} av_dict, file_dict = rollup(avs), rollup(files) for version in versions: v_id = version.id version._compatible_apps = version._compat_map( av_dict.get(v_id, [])) version.all_files = file_dict.get(v_id, []) for f in version.all_files: f.version = version @classmethod def transformer_promoted(cls, versions): """Attach the promoted approvals to the versions.""" if not versions: return PromotedApproval = versions[0].promoted_approvals.model ids = {v.id for v in versions} approvals = list( PromotedApproval.objects.filter(version_id__in=ids).values_list( 'version_id', 'group_id', 'application_id', named=True)) approval_dict = { version_id: list(groups) for version_id, groups in sorted_groupby(approvals, 'version_id') } for version in versions: v_id = version.id groups = [( PROMOTED_GROUPS_BY_ID.get(approval.group_id), APP_IDS.get(approval.application_id), ) for approval in approval_dict.get(v_id, []) if approval.group_id in PROMOTED_GROUPS_BY_ID] version.approved_for_groups = groups @classmethod def transformer_activity(cls, versions): """Attach all the activity to the versions.""" from olympia.activity.models import VersionLog ids = {v.id for v in versions} if not versions: return # Ideally, we'd start from the ActivityLog, but because VersionLog # to ActivityLog isn't a OneToOneField, we wouldn't be able to find # the version easily afterwards - we can't even do a # select_related('versionlog') and try to traverse the relation to find # the version. So, instead, start from VersionLog, but make sure to use # prefetch_related() (and not select_related() - yes, it's one extra # query, but it's worth it to benefit from the default transformer) so # that the ActivityLog default transformer is called. al = (VersionLog.objects.prefetch_related('activity_log').filter( version__in=ids).order_by('created')) def rollup(xs): groups = sorted_groupby(xs, 'version_id') return {k: list(vs) for k, vs in groups} al_dict = rollup(al) for version in versions: v_id = version.id version.all_activity = al_dict.get(v_id, []) @classmethod def transformer_license(cls, versions): """Attach all the licenses to the versions. Do not use if you need the license text: it's explicitly deferred in this transformer, because it should only be used when listing multiple versions, where returning license text is not supposed to be needed. The translations app doesn't fully handle evaluating a deferred field, so the callers need to make sure the license text will never be needed on instances returned by a queryset transformed by this method.""" if not versions: return license_ids = {ver.license_id for ver in versions} licenses = License.objects.filter(id__in=license_ids).defer('text') license_dict = {lic.id: lic for lic in licenses} for version in versions: license = license_dict.get(version.license_id) if license: version.license = license @classmethod def transformer_auto_approvable(cls, versions): """Attach auto-approvability information to the versions.""" ids = {v.id for v in versions} if not ids: return auto_approvable = (Version.objects.auto_approvable().filter( pk__in=ids).values_list('pk', flat=True)) for version in versions: version.is_ready_for_auto_approval = version.pk in auto_approvable def disable_old_files(self): """ Disable files from versions older than the current one in the same channel and awaiting review. Used when uploading a new version. Does nothing if the current instance is unlisted. """ if self.channel == amo.RELEASE_CHANNEL_LISTED: qs = File.objects.filter( version__addon=self.addon_id, version__lt=self.id, version__deleted=False, version__channel=self.channel, status=amo.STATUS_AWAITING_REVIEW, ) # Use File.update so signals are triggered. for f in qs: f.update(status=amo.STATUS_DISABLED) def reset_nomination_time(self, nomination=None): if not self.nomination or nomination: nomination = nomination or datetime.datetime.now() # We need signal=False not to call update_status (which calls us). self.update(nomination=nomination, _signal=False) def inherit_nomination(self, from_statuses=None): last_ver = (Version.objects.filter( addon=self.addon, channel=amo.RELEASE_CHANNEL_LISTED).exclude( nomination=None).exclude(id=self.pk).order_by('-nomination')) if from_statuses: last_ver = last_ver.filter(files__status__in=from_statuses) if last_ver.exists(): self.reset_nomination_time(nomination=last_ver[0].nomination) @property def unreviewed_files(self): """A File is unreviewed if its status is amo.STATUS_AWAITING_REVIEW.""" return self.files.filter(status=amo.STATUS_AWAITING_REVIEW) @cached_property def is_ready_for_auto_approval(self): """Return whether or not this version could be *considered* for auto-approval. Does not necessarily mean that it would be auto-approved, just that it passes the most basic criteria to be considered a candidate by the auto_approve command.""" return Version.objects.auto_approvable().filter(id=self.id).exists() @property def was_auto_approved(self): """Return whether or not this version was auto-approved.""" from olympia.reviewers.models import AutoApprovalSummary try: return (self.is_public() and AutoApprovalSummary.objects.filter( version=self).get().verdict == amo.AUTO_APPROVED) except AutoApprovalSummary.DoesNotExist: pass return False def get_background_images_encoded(self, header_only=False): if not self.has_files: return {} file_obj = self.all_files[0] return { name: force_str(b64encode(background)) for name, background in utils.get_background_images( file_obj, theme_data=None, header_only=header_only).items() } def can_be_disabled_and_deleted(self): # see https://github.com/mozilla/addons-server/issues/15121#issuecomment-667226959 # noqa # "It should apply to the <groups> that require a review to be badged" from olympia.promoted.models import PromotedApproval if self != self.addon.current_version or ( not (group := self.addon.promoted_group()) or not (group.badged and group.pre_review)): return True previous_ver = (self.addon.versions.valid().filter( channel=self.channel).exclude(id=self.id).no_transforms()[:1]) previous_approval = PromotedApproval.objects.filter( group_id=group.id, version__in=previous_ver) return previous_approval.exists()
class Version(OnChangeMixin, ModelBase): id = PositiveAutoField(primary_key=True) addon = models.ForeignKey('addons.Addon', related_name='versions', on_delete=models.CASCADE) license = models.ForeignKey('License', null=True, on_delete=models.CASCADE) release_notes = PurifiedField(db_column='releasenotes', short=False) approval_notes = models.TextField(db_column='approvalnotes', default='', null=True, blank=True) version = models.CharField(max_length=255, default='0.1') version_int = models.BigIntegerField(null=True, editable=False) nomination = models.DateTimeField(null=True) reviewed = models.DateTimeField(null=True) deleted = models.BooleanField(default=False) source = models.FileField(upload_to=source_upload_path, null=True, blank=True) channel = models.IntegerField(choices=amo.RELEASE_CHANNEL_CHOICES, default=amo.RELEASE_CHANNEL_LISTED) git_hash = models.CharField(max_length=40, blank=True) source_git_hash = models.CharField(max_length=40, blank=True) recommendation_approved = models.BooleanField(null=False, default=False) # The order of those managers is very important: please read the lengthy # comment above the Addon managers declaration/instantiation. unfiltered = VersionManager(include_deleted=True) objects = VersionManager() class Meta(ModelBase.Meta): db_table = 'versions' # This is very important: please read the lengthy comment in Addon.Meta # description base_manager_name = 'unfiltered' ordering = ['-created', '-modified'] def __init__(self, *args, **kwargs): super(Version, self).__init__(*args, **kwargs) self.__dict__.update(version_dict(self.version or '')) def __str__(self): return jinja2.escape(self.version) def save(self, *args, **kw): if not self.version_int and self.version: v_int = version_int(self.version) # Magic number warning, this is the maximum size # of a big int in MySQL to prevent version_int overflow, for # people who have rather crazy version numbers. # http://dev.mysql.com/doc/refman/5.5/en/numeric-types.html if v_int < 9223372036854775807: self.version_int = v_int else: log.error('No version_int written for version %s, %s' % (self.pk, self.version)) super(Version, self).save(*args, **kw) return self @classmethod def from_upload(cls, upload, addon, selected_apps, channel, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of compatible app ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ assert parsed_data is not None if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id approval_notes = None if parsed_data.get('is_mozilla_signed_extension'): approval_notes = (u'This version has been signed with ' u'Mozilla internal certificate.') version = cls.objects.create( addon=addon, approval_notes=approval_notes, version=parsed_data['version'], license_id=license_id, channel=channel, ) log.info('New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) if addon.type == amo.ADDON_STATICTHEME: # We don't let developers select apps for static themes selected_apps = [app.id for app in amo.APP_USAGE] compatible_apps = {} for app in parsed_data.get('apps', []): if app.id not in selected_apps: # If the user chose to explicitly deselect Firefox for Android # we're not creating the respective `ApplicationsVersions` # which will have this add-on then be listed only for # Firefox specifically. continue compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version._compatible_apps = compatible_apps # For backwards compatibility. We removed specific platform # support during submission but we don't handle it any different # beyond that yet. That means, we're going to simply set it # to `PLATFORM_ALL` and also have the backend create separate # files for each platform. Cleaning that up is another step. # Given the timing on this, we don't care about updates to legacy # add-ons as well. # Create relevant file and update the all_files cached property on the # Version, because we might need it afterwards. version.all_files = [ File.from_upload(upload=upload, version=version, platform=amo.PLATFORM_ALL.id, parsed_data=parsed_data) ] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) version_uploaded.send(sender=version) # Extract this version into git repository transaction.on_commit( lambda: extract_version_to_git_repository(version, upload)) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): theme_data = parsed_data.get('theme', {}) generate_static_theme_preview(theme_data, version.pk) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version def license_url(self, impala=False): return reverse('addons.license', args=[self.addon.slug, self.version]) def get_url_path(self): if self.channel == amo.RELEASE_CHANNEL_UNLISTED: return '' return reverse('addons.versions', args=[self.addon.slug]) def delete(self, hard=False): # To avoid a circular import from .tasks import delete_preview_files log.info(u'Version deleted: %r (%s)' % (self, self.id)) activity.log_create(amo.LOG.DELETE_VERSION, self.addon, str(self.version)) if hard: super(Version, self).delete() else: # By default we soft delete so we can keep the files for comparison # and a record of the version number. self.files.update(status=amo.STATUS_DISABLED) self.deleted = True self.save() previews_pks = list( VersionPreview.objects.filter(version__id=self.id).values_list( 'id', flat=True)) for preview_pk in previews_pks: delete_preview_files.delay(preview_pk) @property def is_user_disabled(self): return self.files.filter(status=amo.STATUS_DISABLED).exclude( original_status=amo.STATUS_NULL).exists() @is_user_disabled.setter def is_user_disabled(self, disable): # User wants to disable (and the File isn't already). if disable: activity.log_create(amo.LOG.DISABLE_VERSION, self.addon, self) for file in self.files.exclude(status=amo.STATUS_DISABLED).all(): file.update(original_status=file.status, status=amo.STATUS_DISABLED) # User wants to re-enable (and user did the disable, not Mozilla). else: activity.log_create(amo.LOG.ENABLE_VERSION, self.addon, self) for file in self.files.exclude( original_status=amo.STATUS_NULL).all(): file.update(status=file.original_status, original_status=amo.STATUS_NULL) @cached_property def all_activity(self): from olympia.activity.models import VersionLog # yucky al = (VersionLog.objects.filter( version=self.id).order_by('created').select_related( 'activity_log', 'version')) return al @property def compatible_apps(self): # Dicts, search providers and personas don't have compatibility info. # Fake one for them. if self.addon and self.addon.type in amo.NO_COMPAT: return {app: None for app in amo.APP_TYPE_SUPPORT[self.addon.type]} # Otherwise, return _compatible_apps which is a cached property that # is filled by the transformer, or simply calculated from the related # compat instances. return self._compatible_apps @cached_property def _compatible_apps(self): """Get a mapping of {APP: ApplicationsVersions}.""" avs = self.apps.select_related('version') return self._compat_map(avs) @cached_property def compatible_apps_ordered(self): apps = self.compatible_apps.items() return sorted(apps, key=lambda v: v[0].short) def compatible_platforms(self): """Returns a dict of compatible file platforms for this version. The result is based on which app(s) the version targets. """ app_ids = [a.application for a in self.apps.all()] targets_mobile = amo.ANDROID.id in app_ids targets_other = any((id_ != amo.ANDROID.id) for id_ in app_ids) all_plats = {} if targets_other: all_plats.update(amo.DESKTOP_PLATFORMS) if targets_mobile: all_plats.update(amo.MOBILE_PLATFORMS) return all_plats @cached_property def is_compatible_by_default(self): """Returns whether or not the add-on is considered compatible by default.""" # Use self.all_files directly since that's cached and more potentially # prefetched through a transformer already return not any([ file for file in self.all_files if file.binary_components or file.strict_compatibility ]) def is_compatible_app(self, app): """Returns True if the provided app passes compatibility conditions.""" if self.addon.type in amo.NO_COMPAT: return True appversion = self.compatible_apps.get(app) if appversion and app.id in amo.D2C_MIN_VERSIONS: return (version_int(appversion.max.version) >= version_int( amo.D2C_MIN_VERSIONS.get(app.id, '*'))) return False def compat_override_app_versions(self): """Returns the incompatible app versions range(s). If not ranges, returns empty list. Otherwise, this will return all the app version ranges that this particular version is incompatible with. """ overrides = list(self.addon.compatoverride_set.all()) if not overrides: return [] app_versions = [] for co in overrides: for range in co.collapsed_ranges(): if (version_int(range.min) <= version_int(self.version) <= version_int(range.max)): app_versions.extend([(a.min, a.max) for a in range.apps]) return app_versions @cached_property def all_files(self): """Shortcut for list(self.files.all()). Cached.""" return list(self.files.all()) @property def current_file(self): """Shortcut for selecting the first file from self.all_files""" return self.all_files[0] @cached_property def supported_platforms(self): """Get a list of supported platform names.""" return list(set(amo.PLATFORMS[f.platform] for f in self.all_files)) @property def status(self): return [ f.STATUS_CHOICES.get(f.status, ugettext('[status:%s]') % f.status) for f in self.all_files ] @property def statuses(self): """Unadulterated statuses, good for an API.""" return [(f.id, f.status) for f in self.all_files] def is_public(self): # To be public, a version must not be deleted, must belong to a public # addon, and all its attached files must have public status. try: return (not self.deleted and self.addon.is_public() and all(f.status == amo.STATUS_APPROVED for f in self.all_files)) except ObjectDoesNotExist: return False @property def is_restart_required(self): return any(file_.is_restart_required for file_ in self.all_files) @property def is_webextension(self): return any(file_.is_webextension for file_ in self.all_files) @property def is_mozilla_signed(self): """Is the file a special "Mozilla Signed Extension" See https://wiki.mozilla.org/Add-ons/InternalSigning for more details. We use that information to workaround compatibility limits for legacy add-ons and to avoid them receiving negative boosts compared to WebExtensions. See https://github.com/mozilla/addons-server/issues/6424 """ return all(file_.is_mozilla_signed_extension for file_ in self.all_files) @property def has_files(self): return bool(self.all_files) @property def is_unreviewed(self): return bool( list( filter(lambda f: f.status in amo.UNREVIEWED_FILE_STATUSES, self.all_files))) @property def is_all_unreviewed(self): return not bool([ f for f in self.all_files if f.status not in amo.UNREVIEWED_FILE_STATUSES ]) @property def sources_provided(self): return bool(self.source) @classmethod def _compat_map(cls, avs): apps = {} for av in avs: app_id = av.application if app_id in amo.APP_IDS: apps[amo.APP_IDS[app_id]] = av return apps @classmethod def transformer(cls, versions): """Attach all the compatible apps and files to the versions.""" if not versions: return ids = set(v.id for v in versions) avs = (ApplicationsVersions.objects.filter( version__in=ids).select_related('min', 'max')) files = File.objects.filter(version__in=ids) def rollup(xs): groups = sorted_groupby(xs, 'version_id') return dict((k, list(vs)) for k, vs in groups) av_dict, file_dict = rollup(avs), rollup(files) for version in versions: v_id = version.id version._compatible_apps = cls._compat_map(av_dict.get(v_id, [])) version.all_files = file_dict.get(v_id, []) for f in version.all_files: f.version = version @classmethod def transformer_activity(cls, versions): """Attach all the activity to the versions.""" from olympia.activity.models import VersionLog # yucky ids = set(v.id for v in versions) if not versions: return al = (VersionLog.objects.filter( version__in=ids).order_by('created').select_related( 'activity_log', 'version')) def rollup(xs): groups = sorted_groupby(xs, 'version_id') return {k: list(vs) for k, vs in groups} al_dict = rollup(al) for version in versions: v_id = version.id version.all_activity = al_dict.get(v_id, []) def disable_old_files(self): """ Disable files from versions older than the current one and awaiting review. Used when uploading a new version. Does nothing if the current instance is unlisted. """ if self.channel == amo.RELEASE_CHANNEL_LISTED: qs = File.objects.filter( version__addon=self.addon_id, version__lt=self.id, version__deleted=False, status__in=[amo.STATUS_AWAITING_REVIEW, amo.STATUS_PENDING]) # Use File.update so signals are triggered. for f in qs: f.update(status=amo.STATUS_DISABLED) def reset_nomination_time(self, nomination=None): if not self.nomination or nomination: nomination = nomination or datetime.datetime.now() # We need signal=False not to call update_status (which calls us). self.update(nomination=nomination, _signal=False) def inherit_nomination(self, from_statuses=None): last_ver = (Version.objects.filter( addon=self.addon, channel=amo.RELEASE_CHANNEL_LISTED).exclude( nomination=None).exclude(id=self.pk).order_by('-nomination')) if from_statuses: last_ver = last_ver.filter(files__status__in=from_statuses) if last_ver.exists(): self.reset_nomination_time(nomination=last_ver[0].nomination) @property def unreviewed_files(self): """A File is unreviewed if its status is amo.STATUS_AWAITING_REVIEW.""" return self.files.filter(status=amo.STATUS_AWAITING_REVIEW) @property def is_ready_for_auto_approval(self): """Return whether or not this version could be *considered* for auto-approval. Does not necessarily mean that it would be auto-approved, just that it passes the most basic criteria to be considered a candidate by the auto_approve command.""" return Version.objects.auto_approvable().filter(id=self.id).exists() @property def was_auto_approved(self): """Return whether or not this version was auto-approved.""" from olympia.reviewers.models import AutoApprovalSummary try: return self.is_public() and AutoApprovalSummary.objects.filter( version=self).get().verdict == amo.AUTO_APPROVED except AutoApprovalSummary.DoesNotExist: pass return False def get_background_images_encoded(self, header_only=False): if not self.has_files: return {} file_obj = self.all_files[0] return { name: force_text(b64encode(background)) for name, background in utils.get_background_images( file_obj, theme_data=None, header_only=header_only).items() }
class FancyModel(ModelBase): """Mix it up with purified and linkified fields.""" purified = PurifiedField() linkified = LinkifiedField()
class Version(OnChangeMixin, ModelBase): addon = models.ForeignKey('addons.Addon', related_name='versions') license = models.ForeignKey('License', null=True) releasenotes = PurifiedField() approvalnotes = models.TextField(default='', null=True) version = models.CharField(max_length=255, default='0.1') version_int = models.BigIntegerField(null=True, editable=False) nomination = models.DateTimeField(null=True) reviewed = models.DateTimeField(null=True) has_info_request = models.BooleanField(default=False) has_editor_comment = models.BooleanField(default=False) deleted = models.BooleanField(default=False) supported_locales = models.CharField(max_length=255) _developer_name = models.CharField(max_length=255, default='', editable=False) source = models.FileField( upload_to=source_upload_path, null=True, blank=True) # The order of those managers is very important: please read the lengthy # comment above the Addon managers declaration/instanciation. unfiltered = VersionManager(include_deleted=True) objects = VersionManager() class Meta(ModelBase.Meta): db_table = 'versions' ordering = ['-created', '-modified'] def __init__(self, *args, **kwargs): super(Version, self).__init__(*args, **kwargs) self.__dict__.update(version_dict(self.version or '')) def __unicode__(self): return jinja2.escape(self.version) def save(self, *args, **kw): if not self.version_int and self.version: v_int = version_int(self.version) # Magic number warning, this is the maximum size # of a big int in MySQL to prevent version_int overflow, for # people who have rather crazy version numbers. # http://dev.mysql.com/doc/refman/5.5/en/numeric-types.html if v_int < 9223372036854775807: self.version_int = v_int else: log.error('No version_int written for version %s, %s' % (self.pk, self.version)) super(Version, self).save(*args, **kw) return self @classmethod def from_upload(cls, upload, addon, platforms, send_signal=True, source=None, is_beta=False): data = utils.parse_addon(upload, addon) try: license = addon.versions.latest().license_id except Version.DoesNotExist: license = None max_len = cls._meta.get_field_by_name('_developer_name')[0].max_length developer = data.get('developer_name', '')[:max_len] v = cls.objects.create( addon=addon, version=data['version'], license_id=license, _developer_name=developer, source=source ) log.info('New version: %r (%s) from %r' % (v, v.id, upload)) AV = ApplicationsVersions for app in data.get('apps', []): AV(version=v, min=app.min, max=app.max, application=app.id).save() if addon.type == amo.ADDON_SEARCH: # Search extensions are always for all platforms. platforms = [amo.PLATFORM_ALL.id] else: platforms = cls._make_safe_platform_files(platforms) for platform in platforms: File.from_upload(upload, v, platform, parse_data=data, is_beta=is_beta) v.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) if send_signal: version_uploaded.send(sender=v) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}' .format(delta=upload_time, version=v, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return v @classmethod def _make_safe_platform_files(cls, platforms): """Make file platform translations until all download pages support desktop ALL + mobile ALL. See bug 646268. Returns platforms ids. """ pl_set = set(platforms) if pl_set == set((amo.PLATFORM_ALL.id,)): # Make it really ALL: return [amo.PLATFORM_ALL.id] has_mobile = amo.PLATFORM_ANDROID in pl_set has_desktop = any(p in amo.DESKTOP_PLATFORMS for p in pl_set) has_all = amo.PLATFORM_ALL in pl_set is_mixed = has_mobile and has_desktop if (is_mixed and has_all) or has_mobile: # Mixing desktop and mobile w/ ALL is not safe; # we have to split the files into exact platforms. new_plats = [] for platform in platforms: if platform == amo.PLATFORM_ALL.id: plats = amo.DESKTOP_PLATFORMS.keys() plats.remove(amo.PLATFORM_ALL.id) new_plats.extend(plats) else: new_plats.append(platform) return new_plats # Platforms are safe as is return platforms @property def path_prefix(self): return os.path.join(user_media_path('addons'), str(self.addon_id)) @property def mirror_path_prefix(self): return os.path.join(user_media_path('addons'), str(self.addon_id)) def license_url(self, impala=False): return reverse('addons.license', args=[self.addon.slug, self.version]) def flush_urls(self): return self.addon.flush_urls() def get_url_path(self): if not self.addon.is_listed: # Not listed? Doesn't have a public page. return '' return reverse('addons.versions', args=[self.addon.slug, self.version]) def delete(self): log.info(u'Version deleted: %r (%s)' % (self, self.id)) amo.log(amo.LOG.DELETE_VERSION, self.addon, str(self.version)) super(Version, self).delete() @property def current_queue(self): """Return the current queue, or None if not in a queue.""" from olympia.editors.models import ( ViewFullReviewQueue, ViewPendingQueue, ViewPreliminaryQueue, ViewUnlistedFullReviewQueue, ViewUnlistedPendingQueue, ViewUnlistedPreliminaryQueue) if self.addon.status in [amo.STATUS_NOMINATED, amo.STATUS_LITE_AND_NOMINATED]: return (ViewFullReviewQueue if self.addon.is_listed else ViewUnlistedFullReviewQueue) elif self.addon.status == amo.STATUS_PUBLIC: return (ViewPendingQueue if self.addon.is_listed else ViewUnlistedPendingQueue) elif self.addon.status in [amo.STATUS_LITE, amo.STATUS_UNREVIEWED]: return (ViewPreliminaryQueue if self.addon.is_listed else ViewUnlistedPreliminaryQueue) return None @amo.cached_property(writable=True) def all_activity(self): from olympia.devhub.models import VersionLog # yucky al = (VersionLog.objects.filter(version=self.id).order_by('created') .select_related('activity_log', 'version').no_cache()) return al @amo.cached_property(writable=True) def compatible_apps(self): """Get a mapping of {APP: ApplicationVersion}.""" avs = self.apps.select_related('versions', 'license') return self._compat_map(avs) @amo.cached_property def compatible_apps_ordered(self): apps = self.compatible_apps.items() return sorted(apps, key=lambda v: v[0].short) def compatible_platforms(self): """Returns a dict of compatible file platforms for this version. The result is based on which app(s) the version targets. """ app_ids = [a.application for a in self.apps.all()] targets_mobile = amo.ANDROID.id in app_ids targets_other = any((id_ != amo.ANDROID.id) for id_ in app_ids) all_plats = {} if targets_other: all_plats.update(amo.DESKTOP_PLATFORMS) if targets_mobile: all_plats.update(amo.MOBILE_PLATFORMS) return all_plats @amo.cached_property def is_compatible(self): """Returns tuple of compatibility and reasons why if not. Server side conditions for determining compatibility are: * The add-on is an extension (not a theme, app, etc.) * Has not opted in to strict compatibility. * Does not use binary_components in chrome.manifest. Note: The lowest maxVersion compat check needs to be checked separately. Note: This does not take into account the client conditions. """ compat = True reasons = [] if self.addon.type != amo.ADDON_EXTENSION: compat = False # TODO: We may want this. For now we think it may be confusing. # reasons.append(_('Add-on is not an extension.')) if self.files.filter(binary_components=True).exists(): compat = False reasons.append(_('Add-on uses binary components.')) if self.files.filter(strict_compatibility=True).exists(): compat = False reasons.append(_('Add-on has opted into strict compatibility ' 'checking.')) return (compat, reasons) def is_compatible_app(self, app): """Returns True if the provided app passes compatibility conditions.""" appversion = self.compatible_apps.get(app) if appversion and app.id in amo.D2C_MAX_VERSIONS: return (version_int(appversion.max.version) >= version_int(amo.D2C_MAX_VERSIONS.get(app.id, '*'))) return False def compat_override_app_versions(self): """Returns the incompatible app versions range(s). If not ranges, returns empty list. Otherwise, this will return all the app version ranges that this particular version is incompatible with. """ from olympia.addons.models import CompatOverride cos = CompatOverride.objects.filter(addon=self.addon) if not cos: return [] app_versions = [] for co in cos: for range in co.collapsed_ranges(): if (version_int(range.min) <= version_int(self.version) <= version_int(range.max)): app_versions.extend([(a.min, a.max) for a in range.apps]) return app_versions @amo.cached_property(writable=True) def all_files(self): """Shortcut for list(self.files.all()). Heavily cached.""" return list(self.files.all()) @amo.cached_property def supported_platforms(self): """Get a list of supported platform names.""" return list(set(amo.PLATFORMS[f.platform] for f in self.all_files)) @property def status(self): return [f.STATUS_CHOICES[f.status] for f in self.all_files] @property def statuses(self): """Unadulterated statuses, good for an API.""" return [(f.id, f.status) for f in self.all_files] def is_allowed_upload(self): """ Check that a file can be uploaded based on the files per platform for that type of addon. """ num_files = len(self.all_files) if self.addon.type == amo.ADDON_SEARCH: return num_files == 0 elif num_files == 0: return True elif amo.PLATFORM_ALL in self.supported_platforms: return False else: compatible = (v for k, v in self.compatible_platforms().items() if k != amo.PLATFORM_ALL.id) return bool(set(compatible) - set(self.supported_platforms)) def is_public(self): # To be public, a version must not be deleted, must belong to a public # addon, and all its attached files must have public status. try: return (not self.deleted and self.addon.is_public() and all(f.status == amo.STATUS_PUBLIC for f in self.all_files)) except ObjectDoesNotExist: return False @property def has_files(self): return bool(self.all_files) @property def is_unreviewed(self): return filter(lambda f: f.status in amo.UNREVIEWED_STATUSES, self.all_files) @property def is_all_unreviewed(self): return not bool([f for f in self.all_files if f.status not in amo.UNREVIEWED_STATUSES]) @property def is_beta(self): return any(f for f in self.all_files if f.status == amo.STATUS_BETA) @property def is_lite(self): return filter(lambda f: f.status in amo.LITE_STATUSES, self.all_files) @property def is_jetpack(self): return all(f.jetpack_version for f in self.all_files) @classmethod def _compat_map(cls, avs): apps = {} for av in avs: app_id = av.application if app_id in amo.APP_IDS: apps[amo.APP_IDS[app_id]] = av return apps @classmethod def transformer(cls, versions): """Attach all the compatible apps and files to the versions.""" ids = set(v.id for v in versions) if not versions: return # FIXME: find out why we have no_cache() here and try to remove it. avs = (ApplicationsVersions.objects.filter(version__in=ids) .select_related('application', 'apps', 'min_set', 'max_set') .no_cache()) files = File.objects.filter(version__in=ids).no_cache() def rollup(xs): groups = sorted_groupby(xs, 'version_id') return dict((k, list(vs)) for k, vs in groups) av_dict, file_dict = rollup(avs), rollup(files) for version in versions: v_id = version.id version.compatible_apps = cls._compat_map(av_dict.get(v_id, [])) version.all_files = file_dict.get(v_id, []) for f in version.all_files: f.version = version @classmethod def transformer_activity(cls, versions): """Attach all the activity to the versions.""" from olympia.devhub.models import VersionLog # yucky ids = set(v.id for v in versions) if not versions: return al = (VersionLog.objects.filter(version__in=ids).order_by('created') .select_related('activity_log', 'version').no_cache()) def rollup(xs): groups = sorted_groupby(xs, 'version_id') return dict((k, list(vs)) for k, vs in groups) al_dict = rollup(al) for version in versions: v_id = version.id version.all_activity = al_dict.get(v_id, []) def disable_old_files(self): if not self.files.filter(status=amo.STATUS_BETA).exists(): qs = File.objects.filter(version__addon=self.addon_id, version__lt=self.id, version__deleted=False, status__in=[amo.STATUS_UNREVIEWED, amo.STATUS_PENDING]) # Use File.update so signals are triggered. for f in qs: f.update(status=amo.STATUS_DISABLED) @property def developer_name(self): return self._developer_name def reset_nomination_time(self, nomination=None): if not self.nomination or nomination: nomination = nomination or datetime.datetime.now() # We need signal=False not to call update_status (which calls us). self.update(nomination=nomination, _signal=False) # But we need the cache to be flushed. Version.objects.invalidate(self) @property def is_listed(self): return self.addon.is_listed @property def unreviewed_files(self): """A File is unreviewed if: - its status is in amo.UNDER_REVIEW_STATUSES or - its addon status is in amo.UNDER_REVIEW_STATUSES and its status is either in amo.UNDER_REVIEW_STATUSES or amo.STATUS_LITE """ under_review_or_lite = amo.UNDER_REVIEW_STATUSES + (amo.STATUS_LITE,) return self.files.filter( models.Q(status__in=amo.UNDER_REVIEW_STATUSES) | models.Q(version__addon__status__in=amo.UNDER_REVIEW_STATUSES, status__in=under_review_or_lite))
class Version(OnChangeMixin, ModelBase): id = PositiveAutoField(primary_key=True) addon = models.ForeignKey('addons.Addon', related_name='versions', on_delete=models.CASCADE) license = models.ForeignKey('License', null=True, blank=True, on_delete=models.SET_NULL) release_notes = PurifiedField(db_column='releasenotes', short=False) approval_notes = models.TextField(db_column='approvalnotes', default='', null=True, blank=True) version = VersionStringField(max_length=255, default='0.1') nomination = models.DateTimeField(null=True) reviewed = models.DateTimeField(null=True) deleted = models.BooleanField(default=False) source = models.FileField( upload_to=source_upload_path, storage=source_upload_storage, null=True, blank=True, max_length=255, ) channel = models.IntegerField(choices=amo.RELEASE_CHANNEL_CHOICES, default=amo.RELEASE_CHANNEL_LISTED) git_hash = models.CharField(max_length=40, blank=True) needs_human_review = models.BooleanField(default=False) # The order of those managers is very important: please read the lengthy # comment above the Addon managers declaration/instantiation. unfiltered = VersionManager(include_deleted=True) objects = VersionManager() # See UnfilteredVersionManagerForRelations() docstring for usage of this # special manager. unfiltered_for_relations = UnfilteredVersionManagerForRelations() class Meta(ModelBase.Meta): db_table = 'versions' # This is very important: please read the lengthy comment in Addon.Meta # description base_manager_name = 'unfiltered' ordering = ['-created', '-modified'] indexes = [ models.Index(fields=('addon', ), name='addon_id'), models.Index(fields=('license', ), name='license_id'), ] constraints = [ models.UniqueConstraint( fields=('addon', 'version'), name='versions_addon_id_version_5a2e75b6_uniq', ), ] def __str__(self): return markupsafe.escape(self.version) @classmethod def from_upload( cls, upload, addon, channel, *, selected_apps=None, compatibility=None, parsed_data=None, ): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a channel id and the parsed_data generated by parse_addon(). Additionally, for non-themes: either a list of compatible app ids needs to be provided as `selected_apps`, or a list of `ApplicationVersions` instances for each compatible app as `compatibility`. If `compatibility` is provided: the `version` property of the instances will be set to the new upload and the instances saved. If the min and/or max properties of the `ApplicationVersions` instance are none then `AppVersion`s parsed from the manifest, or defaults, are used. Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ from olympia.addons.models import AddonReviewerFlags from olympia.addons.utils import RestrictionChecker from olympia.git.utils import create_git_extraction_entry assert parsed_data is not None if addon.type == amo.ADDON_STATICTHEME: # We don't let developers select apps for static themes compatibility = { app: (compatibility or {}).get(app, ApplicationsVersions(application=app.id)) for app in amo.APP_USAGE } assert selected_apps or compatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') if upload.addon and upload.addon != addon: raise VersionCreateError( 'FileUpload was made for a different Addon') if (not getattr(upload, 'user', None) or not upload.ip_address or not upload.source): raise VersionCreateError( 'FileUpload does not have some required fields') if not upload.user.last_login_ip or not upload.user.email: raise VersionCreateError( 'FileUpload user does not have some required fields') # This should be guaranteed by the linter, just raise an explicit # exception if somehow it's wrong. if not isinstance(parsed_data.get('install_origins', []), list): raise VersionCreateError( 'install_origins was not validated properly') license_id = parsed_data.get('license_id') if not license_id and channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id approval_notes = None if parsed_data.get('is_mozilla_signed_extension'): approval_notes = ( 'This version has been signed with Mozilla internal certificate.' ) version = cls.objects.create( addon=addon, approval_notes=approval_notes, version=parsed_data['version'], license_id=license_id, channel=channel, release_notes=parsed_data.get('release_notes'), ) with core.override_remote_addr(upload.ip_address): # The following log statement is used by foxsec-pipeline. # We override the IP because it might be called from a task and we # want the original IP from the submitter. log.info( f'New version: {version!r} ({version.id}) from {upload!r}', extra={ 'email': upload.user.email, 'guid': addon.guid, 'upload': upload.uuid.hex, 'user_id': upload.user_id, 'from_api': upload.source == amo.UPLOAD_SOURCE_SIGNING_API, }, ) activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=upload.user) if not compatibility: compatibility = { amo.APP_IDS[app_id]: ApplicationsVersions(application=app_id) for app_id in selected_apps } compatible_apps = {} for parsed_app in parsed_data.get('apps', []): if parsed_app.appdata not in compatibility: # If the user chose to explicitly deselect Firefox for Android # we're not creating the respective `ApplicationsVersions` # which will have this add-on then be listed only for # Firefox specifically. continue avs = compatibility[parsed_app.appdata] avs.version = version avs.min = getattr(avs, 'min', parsed_app.min) avs.max = getattr(avs, 'max', parsed_app.max) avs.save() compatible_apps[parsed_app.appdata] = avs # Pre-generate compatible_apps property to avoid accidentally # triggering queries with that instance later. version.compatible_apps = compatible_apps # Record declared install origins. base_domain is set automatically. if waffle.switch_is_active('record-install-origins'): for origin in set(parsed_data.get('install_origins', [])): version.installorigin_set.create(origin=origin) # Create relevant file. File.from_upload( upload=upload, version=version, parsed_data=parsed_data, ) version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to its permanent location, delete it # from storage. Keep the FileUpload instance (it gets cleaned up by a # cron eventually some time after its creation, in amo.cron.gc()), # making sure it's associated with the add-on instance. storage.delete(upload.path) upload.path = '' if upload.addon is None: upload.addon = addon upload.save() version_uploaded.send(instance=version, sender=Version) if (waffle.switch_is_active('enable-yara') or waffle.switch_is_active('enable-customs') or waffle.switch_is_active('enable-wat')): ScannerResult.objects.filter(upload_id=upload.id).update( version=version) if waffle.switch_is_active('enable-uploads-commit-to-git-storage'): # Schedule this version for git extraction. transaction.on_commit( lambda: create_git_extraction_entry(version=version)) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): theme_data = parsed_data.get('theme', {}) generate_static_theme_preview(theme_data, version.pk) # Reset add-on reviewer flags to disable auto-approval and require # admin code review if the package has already been signed by mozilla. reviewer_flags_defaults = {} is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension') if upload.validation_timeout: reviewer_flags_defaults['needs_admin_code_review'] = True if is_mozilla_signed and addon.type != amo.ADDON_LPAPP: reviewer_flags_defaults['needs_admin_code_review'] = True reviewer_flags_defaults['auto_approval_disabled'] = True # Check if the approval should be restricted if not RestrictionChecker(upload=upload).is_auto_approval_allowed(): flag = ('auto_approval_disabled' if channel == amo.RELEASE_CHANNEL_LISTED else 'auto_approval_disabled_unlisted') reviewer_flags_defaults[flag] = True if reviewer_flags_defaults: AddonReviewerFlags.objects.update_or_create( addon=addon, defaults=reviewer_flags_defaults) # Authors need to be notified about auto-approval delay again since # they are submitting a new version. addon.reset_notified_about_auto_approval_delay() # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version def license_url(self, impala=False): return reverse('addons.license', args=[self.addon.slug, self.version]) def get_url_path(self): if self.channel == amo.RELEASE_CHANNEL_UNLISTED: return '' return reverse('addons.versions', args=[self.addon.slug]) def delete(self, hard=False): # To avoid a circular import from .tasks import delete_preview_files log.info(f'Version deleted: {self!r} ({self.id})') activity.log_create(amo.LOG.DELETE_VERSION, self.addon, str(self.version)) if hard: super().delete() else: # By default we soft delete so we can keep the files for comparison # and a record of the version number. if hasattr(self, 'file'): # .file should always exist but we don't want to break delete regardless self.file.update(status=amo.STATUS_DISABLED) self.deleted = True self.save() # Clear pending rejection flag (we have the activity log for # records purposes, the flag serves no purpose anymore if the # version is deleted). VersionReviewerFlags.objects.filter(version=self).update( pending_rejection=None, pending_rejection_by=None) previews_pks = list( VersionPreview.objects.filter(version__id=self.id).values_list( 'id', flat=True)) for preview_pk in previews_pks: delete_preview_files.delay(preview_pk) @property def is_user_disabled(self): return (self.file.status == amo.STATUS_DISABLED and self.file.original_status != amo.STATUS_NULL) @is_user_disabled.setter def is_user_disabled(self, disable): # User wants to disable (and the File isn't already). if disable: activity.log_create(amo.LOG.DISABLE_VERSION, self.addon, self) if (file_ := self.file) and file_.status != amo.STATUS_DISABLED: file_.update(original_status=file_.status, status=amo.STATUS_DISABLED) # User wants to re-enable (and user did the disable, not Mozilla). else: