def migrate_legacy_dictionary_to_webextension(addon): """Migrate a single legacy dictionary to webextension format, creating a new package from the current_version, faking an upload to create a new Version instance.""" user = UserProfile.objects.get(pk=settings.TASK_USER_ID) now = datetime.now() # Wrap zip in FileUpload for Version.from_upload() to consume. upload = FileUpload.objects.create( user=user, valid=True) destination = os.path.join( user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') target_language = build_webext_dictionary_from_legacy(addon, destination) if not addon.target_locale: addon.update(target_locale=target_language) upload.update(path=destination) parsed_data = parse_addon(upload, addon=addon, user=user) # Create version. # WebExtension dictionaries are only compatible with Firefox Desktop # Firefox for Android uses the OS spellchecking. version = Version.from_upload( upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user) # Sign the file, and set it to public. That should automatically set # current_version to the version we created. file_ = version.all_files[0] sign_file(file_) file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def delete(self, user_responsible=None): if user_responsible is None: user_responsible = self.user rating_was_moderated = False # Log deleting ratings to moderation log, # except if the author deletes it if user_responsible != self.user: # Remember moderation state rating_was_moderated = True from olympia.reviewers.models import ReviewerScore activity.log_create( amo.LOG.DELETE_RATING, self.addon, self, user=user_responsible, details=dict( body=unicode(self.body), addon_id=self.addon.pk, addon_title=unicode(self.addon.name), is_flagged=self.ratingflag_set.exists())) for flag in self.ratingflag_set.all(): flag.delete() log.info(u'Rating deleted: %s deleted id:%s by %s ("%s")', user_responsible.name, self.pk, self.user.name, unicode(self.body)) self.update(deleted=True) # Force refreshing of denormalized data (it wouldn't happen otherwise # because we're not dealing with a creation). self.update_denormalized_fields() if rating_was_moderated: ReviewerScore.award_moderation_points(user_responsible, self.addon, self.pk)
def post_save(sender, instance, created, **kwargs): if kwargs.get('raw'): return if getattr(instance, 'user_responsible', None): # user_responsible is not a field on the model, so it's not # persistent: it's just something the views will set temporarily # when manipulating a Rating that indicates a real user made that # change. action = 'New' if created else 'Edited' if instance.reply_to: log.debug('%s reply to %s: %s' % ( action, instance.reply_to_id, instance.pk)) else: log.debug('%s rating: %s' % (action, instance.pk)) # For new ratings - not replies - and all edits (including replies # this time) by users we want to insert a new ActivityLog. new_rating_or_edit = not instance.reply_to or not created if new_rating_or_edit: action = amo.LOG.ADD_RATING if created else amo.LOG.EDIT_RATING activity.log_create(action, instance.addon, instance, user=instance.user_responsible) # For new ratings and new replies we want to send an email. if created: instance.send_notification_email() instance.refresh(update_denorm=created)
def groupuser_post_delete(sender, instance, **kw): if kw.get('raw'): return activity.log_create(amo.LOG.GROUP_USER_REMOVED, instance.group, instance.user) log.info('Removed %s from %s' % (instance.user, instance.group)) instance.invalidate_groups_list()
def groupuser_post_save(sender, instance, **kw): if kw.get('raw'): return activity.log_create(amo.LOG.GROUP_USER_ADDED, instance.group, instance.user) log.info('Added %s to %s' % (instance.user, instance.group)) instance.invalidate_groups_list()
def delete(self, hard=False): log.info(u'Version deleted: %r (%s)' % (self, self.id)) activity.log_create(amo.LOG.DELETE_VERSION, self.addon, str(self.version)) if hard: super(Version, self).delete() else: # By default we soft delete so we can keep the files for comparison # and a record of the version number. self.files.update(status=amo.STATUS_DISABLED) self.deleted = True self.save()
def set_addons(self, addon_ids, comments=None): """Replace the current add-ons with a new list of add-on ids.""" if comments is None: comments = {} order = {a: idx for idx, a in enumerate(addon_ids)} # Partition addon_ids into add/update/remove buckets. existing = set(self.addons.using('default') .values_list('id', flat=True)) add, update = [], [] for addon in addon_ids: bucket = update if addon in existing else add bucket.append((addon, order[addon])) remove = existing.difference(addon_ids) now = datetime.now() with connection.cursor() as cursor: if remove: cursor.execute("DELETE FROM addons_collections " "WHERE collection_id=%s AND addon_id IN (%s)" % (self.id, ','.join(map(str, remove)))) if self.listed: for addon in remove: activity.log_create(amo.LOG.REMOVE_FROM_COLLECTION, (Addon, addon), self) if add: insert = '(%s, %s, %s, NOW(), NOW(), 0)' values = [insert % (a, self.id, idx) for a, idx in add] cursor.execute(""" INSERT INTO addons_collections (addon_id, collection_id, ordering, created, modified, downloads) VALUES %s""" % ','.join(values)) if self.listed: for addon_id, idx in add: activity.log_create(amo.LOG.ADD_TO_COLLECTION, (Addon, addon_id), self) for addon, ordering in update: (CollectionAddon.objects.filter(collection=self.id, addon=addon) .update(ordering=ordering, modified=now)) for addon, comment in comments.iteritems(): try: c = (CollectionAddon.objects.using('default') .get(collection=self.id, addon=addon)) except CollectionAddon.DoesNotExist: pass else: c.comments = comment c.save(force_update=True) self.save()
def add_static_theme_from_lwt(lwt): # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create( user=author, valid=True) destination = os.path.join( user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination) # Create addon + version parsed_data = parse_addon(upload, user=author) addon = Addon.initialize_addon_from_upload( parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) # Version.from_upload sorts out platforms for us. version = Version.from_upload( upload, addon, platforms=None, channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) # Set category static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get( amo.ADDON_STATICTHEME, []) lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create( addon=addon, category=Category.from_static_category(static_category, True)) # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) # Logging activity.log_create( amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) log.debug('New static theme %r created from %r' % (addon, lwt)) # And finally update the statuses version.all_files[0].update(status=amo.STATUS_PUBLIC) addon.update(status=amo.STATUS_PUBLIC) return addon
def is_user_disabled(self, disable): # User wants to disable (and the File isn't already). if disable: activity.log_create(amo.LOG.DISABLE_VERSION, self.addon, self) for file in self.files.exclude(status=amo.STATUS_DISABLED).all(): file.update(original_status=file.status, status=amo.STATUS_DISABLED) # User wants to re-enable (and user did the disable, not Mozilla). else: activity.log_create(amo.LOG.ENABLE_VERSION, self.addon, self) for file in self.files.exclude( original_status=amo.STATUS_NULL).all(): file.update(status=file.original_status, original_status=amo.STATUS_NULL)
def approve(self, user): from olympia.reviewers.models import ReviewerScore activity.log_create( amo.LOG.APPROVE_RATING, self.addon, self, user=user, details=dict( body=unicode(self.body), addon_id=self.addon.pk, addon_title=unicode(self.addon.name), is_flagged=self.ratingflag_set.exists())) for flag in self.ratingflag_set.all(): flag.delete() self.editorreview = False # We've already logged what we want to log, no need to pass # user_responsible=user. self.save() ReviewerScore.award_moderation_points(user, self.addon, self.pk)
def delete(self, hard=False): # To avoid a circular import from .tasks import delete_preview_files log.info(u'Version deleted: %r (%s)' % (self, self.id)) activity.log_create(amo.LOG.DELETE_VERSION, self.addon, str(self.version)) if hard: super(Version, self).delete() else: # By default we soft delete so we can keep the files for comparison # and a record of the version number. self.files.update(status=amo.STATUS_DISABLED) self.deleted = True self.save() previews_pks = list( VersionPreview.objects.filter(version__id=self.id) .values_list('id', flat=True)) for preview_pk in previews_pks: delete_preview_files.delay(preview_pk)
def remove_addon(self, addon): CollectionAddon.objects.filter(addon=addon, collection=self).delete() if self.listed: activity.log_create(amo.LOG.REMOVE_FROM_COLLECTION, addon, self) self.save() # To invalidate Collection.
def save_tag(self, addon): tag, created = Tag.objects.get_or_create(tag_text=self.tag_text) AddonTag.objects.get_or_create(addon=addon, tag=tag) activity.log_create(amo.LOG.ADD_TAG, tag, addon) return tag
def add_static_theme_from_lwt(lwt): from olympia.activity.models import AddonLog olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID)) # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create( user=author, valid=True) destination = os.path.join( user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination) # Create addon + version parsed_data = parse_addon(upload, user=author) addon = Addon.initialize_addon_from_upload( parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) addon_updates = {} # static themes are only compatible with Firefox at the moment, # not Android version = Version.from_upload( upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) # Set category static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get( amo.ADDON_STATICTHEME, []) lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create( addon=addon, category=Category.from_static_category(static_category, True)) # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) # Steal the ratings (even with soft delete they'll be deleted anyway) addon_updates.update( average_rating=lwt.average_rating, bayesian_rating=lwt.bayesian_rating, total_ratings=lwt.total_ratings, text_ratings_count=lwt.text_ratings_count) Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version) # Modify the activity log entry too. rating_activity_log_ids = [ l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review'] addonlog_qs = AddonLog.objects.filter( addon=lwt, activity_log__action__in=rating_activity_log_ids) [alog.transfer(addon) for alog in addonlog_qs.iterator()] # Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats # dashboard and future update counts, and copy the summary numbers for now. migrate_theme_update_count(lwt, addon) addon_updates.update( average_daily_users=lwt.persona.popularity or 0, hotness=lwt.persona.movers or 0) # Logging activity.log_create( amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update( datestatuschanged=lwt.last_updated, reviewed=datetime.now(), status=amo.STATUS_PUBLIC) addon_updates['status'] = amo.STATUS_PUBLIC # set the modified and creation dates to match the original. addon_updates['created'] = lwt.created addon_updates['modified'] = lwt.modified addon_updates['last_updated'] = lwt.last_updated addon.update(**addon_updates) return addon
def remove_tag(self, addon): tag, created = Tag.objects.get_or_create(tag_text=self.tag_text) for addon_tag in AddonTag.objects.filter(addon=addon, tag=tag): addon_tag.delete() activity.log_create(amo.LOG.REMOVE_TAG, tag, addon)
def from_upload(cls, upload, addon, platforms, channel, send_signal=True, source=None, is_beta=False, parsed_data=None): from olympia.addons.models import AddonFeatureCompatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') if parsed_data is None: parsed_data = utils.parse_addon(upload, addon) license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version( channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id version = cls.objects.create( addon=addon, version=parsed_data['version'], license_id=license_id, source=source, channel=channel, ) log.info( 'New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) # Update the add-on e10s compatibility since we're creating a new # version that may change that. e10s_compatibility = parsed_data.get('e10s_compatibility') if e10s_compatibility is not None: feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0] ) feature_compatibility.update(e10s=e10s_compatibility) compatible_apps = {} for app in parsed_data.get('apps', []): compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version.compatible_apps = compatible_apps if addon.type == amo.ADDON_SEARCH: # Search extensions are always for all platforms. platforms = [amo.PLATFORM_ALL.id] else: platforms = cls._make_safe_platform_files(platforms) for platform in platforms: File.from_upload(upload, version, platform, parsed_data=parsed_data, is_beta=is_beta) version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) if send_signal: version_uploaded.send(sender=version) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}' .format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version
def from_upload(cls, upload, addon, platforms, channel, send_signal=True, source=None, is_beta=False, parsed_data=None): from olympia.addons.models import AddonFeatureCompatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') if parsed_data is None: parsed_data = utils.parse_addon(upload, addon) license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id version = cls.objects.create( addon=addon, version=parsed_data['version'], license_id=license_id, source=source, channel=channel, ) log.info('New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) # Update the add-on e10s compatibility since we're creating a new # version that may change that. e10s_compatibility = parsed_data.get('e10s_compatibility') if e10s_compatibility is not None: feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create( addon=addon)[0]) feature_compatibility.update(e10s=e10s_compatibility) compatible_apps = {} for app in parsed_data.get('apps', []): compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version.compatible_apps = compatible_apps if addon.type == amo.ADDON_SEARCH: # Search extensions are always for all platforms. platforms = [amo.PLATFORM_ALL.id] else: platforms = cls._make_safe_platform_files(platforms) for platform in platforms: File.from_upload(upload, version, platform, parsed_data=parsed_data, is_beta=is_beta) version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) if send_signal: version_uploaded.send(sender=version) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version
def add_addon(self, addon): "Adds an addon to the collection." CollectionAddon.objects.get_or_create(addon=addon, collection=self) if self.listed: activity.log_create(amo.LOG.ADD_TO_COLLECTION, addon, self) self.save() # To invalidate Collection.
def from_upload(cls, upload, addon, selected_apps, channel, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of compatible app ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ from olympia.git.utils import create_git_extraction_entry assert parsed_data is not None if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id approval_notes = None if parsed_data.get('is_mozilla_signed_extension'): approval_notes = (u'This version has been signed with ' u'Mozilla internal certificate.') version = cls.objects.create( addon=addon, approval_notes=approval_notes, version=parsed_data['version'], license_id=license_id, channel=channel, ) email = upload.user.email if upload.user and upload.user.email else '' with core.override_remote_addr(upload.ip_address): log.info('New version: %r (%s) from %r' % (version, version.id, upload), extra={ 'email': email, 'guid': addon.guid, 'upload': upload.uuid.hex, 'user_id': upload.user_id, 'from_api': upload.source == amo.UPLOAD_SOURCE_API, }) activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=upload.user or get_task_user()) if addon.type == amo.ADDON_STATICTHEME: # We don't let developers select apps for static themes selected_apps = [app.id for app in amo.APP_USAGE] compatible_apps = {} for app in parsed_data.get('apps', []): if app.id not in selected_apps: # If the user chose to explicitly deselect Firefox for Android # we're not creating the respective `ApplicationsVersions` # which will have this add-on then be listed only for # Firefox specifically. continue compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version._compatible_apps = compatible_apps # For backwards compatibility. We removed specific platform # support during submission but we don't handle it any different # beyond that yet. That means, we're going to simply set it # to `PLATFORM_ALL` and also have the backend create separate # files for each platform. Cleaning that up is another step. # Given the timing on this, we don't care about updates to legacy # add-ons as well. # Create relevant file and update the all_files cached property on the # Version, because we might need it afterwards. version.all_files = [ File.from_upload(upload=upload, version=version, platform=amo.PLATFORM_ALL.id, parsed_data=parsed_data) ] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) upload.path = '' upload.save() version_uploaded.send(instance=version, sender=Version) if version.is_webextension: if (waffle.switch_is_active('enable-yara') or waffle.switch_is_active('enable-customs') or waffle.switch_is_active('enable-wat')): ScannerResult.objects.filter(upload_id=upload.id).update( version=version) if waffle.switch_is_active('enable-uploads-commit-to-git-storage'): # Schedule this version for git extraction. transaction.on_commit( lambda: create_git_extraction_entry(version=version)) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): theme_data = parsed_data.get('theme', {}) generate_static_theme_preview(theme_data, version.pk) # Authors need to be notified about auto-approval delay again since # they are submitting a new version. addon.reset_notified_about_auto_approval_delay() # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version
def from_upload(cls, upload, addon, platforms, channel, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of platform ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ assert parsed_data is not None from olympia.addons.models import AddonFeatureCompatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id version = cls.objects.create( addon=addon, version=parsed_data['version'], license_id=license_id, channel=channel, ) log.info('New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) # Update the add-on e10s compatibility since we're creating a new # version that may change that. e10s_compatibility = parsed_data.get('e10s_compatibility') if e10s_compatibility is not None: feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create( addon=addon)[0]) feature_compatibility.update(e10s=e10s_compatibility) compatible_apps = {} for app in parsed_data.get('apps', []): compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version._compatible_apps = compatible_apps if addon.type in [amo.ADDON_SEARCH, amo.ADDON_STATICTHEME]: # Search extensions and static themes are always for all platforms. platforms = [amo.PLATFORM_ALL.id] else: platforms = cls._make_safe_platform_files(platforms) # Create as many files as we have platforms. Update the all_files # cached property on the Version while we're at it, because we might # need it afterwards. version.all_files = [ File.from_upload(upload, version, platform, parsed_data=parsed_data) for platform in platforms ] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) version_uploaded.send(sender=version) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): dst_root = os.path.join(user_media_path('addons'), str(addon.id)) theme_data = parsed_data.get('theme', {}) version_root = os.path.join(dst_root, unicode(version.id)) utils.extract_header_img(version.all_files[0].file_path, theme_data, version_root) generate_static_theme_preview(theme_data, version_root, version.pk) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version
def add_static_theme_from_lwt(lwt): from olympia.activity.models import AddonLog # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create(user=author, valid=True) destination = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination) # Create addon + version parsed_data = parse_addon(upload, user=author) addon = Addon.initialize_addon_from_upload(parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) addon_updates = {} # static themes are only compatible with Firefox at the moment, # not Android version = Version.from_upload(upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) # Set category static_theme_categories = CATEGORIES.get(amo.THUNDERBIRD.id, []).get(amo.ADDON_STATICTHEME, []) lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create(addon=addon, category=Category.from_static_category( static_category, True)) # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) # Steal the ratings (even with soft delete they'll be deleted anyway) addon_updates.update(average_rating=lwt.average_rating, bayesian_rating=lwt.bayesian_rating, total_ratings=lwt.total_ratings, text_ratings_count=lwt.text_ratings_count) Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version) # Modify the activity log entry too. rating_activity_log_ids = [ l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review' ] addonlog_qs = AddonLog.objects.filter( addon=lwt, activity_log__action__in=rating_activity_log_ids) [alog.transfer(addon) for alog in addonlog_qs.iterator()] # Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats # dashboard and future update counts, and copy the summary numbers for now. migrate_theme_update_count(lwt, addon) addon_updates.update(average_daily_users=lwt.persona.popularity or 0, hotness=lwt.persona.movers or 0) # Logging activity.log_create(amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update(datestatuschanged=lwt.last_updated, reviewed=datetime.now(), status=amo.STATUS_PUBLIC) addon_updates['status'] = amo.STATUS_PUBLIC # set the modified and creation dates to match the original. addon_updates['created'] = lwt.created addon_updates['modified'] = lwt.modified addon_updates['last_updated'] = lwt.last_updated addon.update(**addon_updates) return addon
def add_static_theme_from_lwt(lwt): from olympia.activity.models import AddonLog timer = StopWatch( 'addons.tasks.migrate_lwts_to_static_theme.add_from_lwt.') timer.start() olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID)) # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create( user=author, valid=True) filename = uuid.uuid4().hex + '.xpi' destination = os.path.join(user_media_path('addons'), 'temp', filename) build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination, name=filename) timer.log_interval('1.build_xpi') # Create addon + version parsed_data = parse_addon(upload, user=author) timer.log_interval('2a.parse_addon') addon = Addon.initialize_addon_from_upload( parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) addon_updates = {} timer.log_interval('2b.initialize_addon') # static themes are only compatible with Firefox at the moment, # not Android version = Version.from_upload( upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) timer.log_interval('3.initialize_version') # Set category lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' for app, type_dict in CATEGORIES.items(): static_theme_categories = type_dict.get(amo.ADDON_STATICTHEME, []) static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create( addon=addon, category=Category.from_static_category(static_category, True)) timer.log_interval('4.set_categories') # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) timer.log_interval('5.set_license') # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) timer.log_interval('6.set_tags') # Steal the ratings (even with soft delete they'll be deleted anyway) addon_updates.update( average_rating=lwt.average_rating, bayesian_rating=lwt.bayesian_rating, total_ratings=lwt.total_ratings, text_ratings_count=lwt.text_ratings_count) Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version) timer.log_interval('7.move_ratings') # Replace the lwt in collections CollectionAddon.objects.filter(addon=lwt).update(addon=addon) # Modify the activity log entry too. rating_activity_log_ids = [ l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review'] addonlog_qs = AddonLog.objects.filter( addon=lwt, activity_log__action__in=rating_activity_log_ids) [alog.transfer(addon) for alog in addonlog_qs.iterator()] timer.log_interval('8.move_activity_logs') # Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats # dashboard and future update counts, and copy the average_daily_users. # hotness will be recalculated by the deliver_hotness() cron in a more # reliable way that we could do, so skip it entirely. migrate_theme_update_count(lwt, addon) addon_updates.update( average_daily_users=lwt.persona.popularity or 0, hotness=0) timer.log_interval('9.copy_statistics') # Logging activity.log_create( amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update( datestatuschanged=lwt.last_updated, reviewed=datetime.now(), status=amo.STATUS_PUBLIC) timer.log_interval('10.sign_files') addon_updates['status'] = amo.STATUS_PUBLIC # set the modified and creation dates to match the original. addon_updates['created'] = lwt.created addon_updates['modified'] = lwt.modified addon_updates['last_updated'] = lwt.last_updated addon.update(**addon_updates) return addon
def from_upload(cls, upload, addon, selected_apps, channel, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of compatible app ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ from olympia.addons.models import AddonReviewerFlags from olympia.git.utils import create_git_extraction_entry assert parsed_data is not None if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.' ) if upload.addon and upload.addon != addon: raise VersionCreateError('FileUpload was made for a different Addon') if not upload.user or not upload.ip_address or not upload.source: raise VersionCreateError('FileUpload does not have some required fields') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version(channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id approval_notes = None if parsed_data.get('is_mozilla_signed_extension'): approval_notes = ( 'This version has been signed with Mozilla internal certificate.' ) version = cls.objects.create( addon=addon, approval_notes=approval_notes, version=parsed_data['version'], license_id=license_id, channel=channel, ) email = upload.user.email if upload.user and upload.user.email else '' with core.override_remote_addr(upload.ip_address): # The following log statement is used by foxsec-pipeline. # We override the IP because it might be called from a task and we # want the original IP from the submitter. log.info( 'New version: %r (%s) from %r' % (version, version.id, upload), extra={ 'email': email, 'guid': addon.guid, 'upload': upload.uuid.hex, 'user_id': upload.user_id, 'from_api': upload.source == amo.UPLOAD_SOURCE_API, }, ) activity.log_create( amo.LOG.ADD_VERSION, version, addon, user=upload.user or get_task_user() ) if addon.type == amo.ADDON_STATICTHEME: # We don't let developers select apps for static themes selected_apps = [app.id for app in amo.APP_USAGE] compatible_apps = {} for app in parsed_data.get('apps', []): if app.id not in selected_apps: # If the user chose to explicitly deselect Firefox for Android # we're not creating the respective `ApplicationsVersions` # which will have this add-on then be listed only for # Firefox specifically. continue compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id ) compatible_apps[app.appdata].save() # Pre-generate _compatible_apps property to avoid accidentally # triggering queries with that instance later. version._compatible_apps = compatible_apps # Create relevant file and update the all_files cached property on the # Version, because we might need it afterwards. version.all_files = [ File.from_upload( upload=upload, version=version, parsed_data=parsed_data, ) ] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to its permanent location, delete it # from storage. Keep the FileUpload instance (it gets cleaned up by a # cron eventually some time after its creation, in amo.cron.gc()), # making sure it's associated with the add-on instance. storage.delete(upload.path) upload.path = '' if upload.addon is None: upload.addon = addon upload.save() version_uploaded.send(instance=version, sender=Version) if version.is_webextension: if ( waffle.switch_is_active('enable-yara') or waffle.switch_is_active('enable-customs') or waffle.switch_is_active('enable-wat') ): ScannerResult.objects.filter(upload_id=upload.id).update( version=version ) if waffle.switch_is_active('enable-uploads-commit-to-git-storage'): # Schedule this version for git extraction. transaction.on_commit(lambda: create_git_extraction_entry(version=version)) # Generate a preview and icon for listed static themes if ( addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED ): theme_data = parsed_data.get('theme', {}) generate_static_theme_preview(theme_data, version.pk) # Reset add-on reviewer flags to disable auto-approval and require # admin code review if the package has already been signed by mozilla. reviewer_flags_defaults = {} is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension') if upload.validation_timeout: reviewer_flags_defaults['needs_admin_code_review'] = True if is_mozilla_signed and addon.type != amo.ADDON_LPAPP: reviewer_flags_defaults['needs_admin_code_review'] = True reviewer_flags_defaults['auto_approval_disabled'] = True if reviewer_flags_defaults: AddonReviewerFlags.objects.update_or_create( addon=addon, defaults=reviewer_flags_defaults ) # Authors need to be notified about auto-approval delay again since # they are submitting a new version. addon.reset_notified_about_auto_approval_delay() # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info( 'Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}'.format( delta=upload_time, version=version, created=upload.created, now=now ) ) statsd.timing('devhub.version_created_from_upload', upload_time) return version
def from_upload(cls, upload, addon, selected_apps, channel, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of compatible app ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ assert parsed_data is not None from olympia.addons.models import AddonFeatureCompatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version( channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id approvalnotes = None if parsed_data.get('is_mozilla_signed_extension'): approvalnotes = (u'This version has been signed with ' u'Mozilla internal certificate.') version = cls.objects.create( addon=addon, approvalnotes=approvalnotes, version=parsed_data['version'], license_id=license_id, channel=channel, ) log.info( 'New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) # Update the add-on e10s compatibility since we're creating a new # version that may change that. e10s_compatibility = parsed_data.get('e10s_compatibility') if e10s_compatibility is not None: feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0] ) feature_compatibility.update(e10s=e10s_compatibility) compatible_apps = {} for app in parsed_data.get('apps', []): if app.id not in selected_apps: # If the user chose to explicitly deselect Firefox for Android # we're not creating the respective `ApplicationsVersions` # which will have this add-on then be listed only for # Firefox specifically. continue compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version._compatible_apps = compatible_apps # For backwards compatibility. We removed specific platform # support during submission but we don't handle it any different # beyond that yet. That means, we're going to simply set it # to `PLATFORM_ALL` and also have the backend create separate # files for each platform. Cleaning that up is another step. # Given the timing on this, we don't care about updates to legacy # add-ons as well. # Create relevant file and update the all_files cached property on the # Version, because we might need it afterwards. version.all_files = [File.from_upload( upload=upload, version=version, platform=amo.PLATFORM_ALL.id, parsed_data=parsed_data )] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) version_uploaded.send(sender=version) if waffle.switch_is_active('enable-uploads-commit-to-git-storage'): # Extract into git repository AddonGitRepository.extract_and_commit_from_file_obj( file_obj=version.all_files[0], channel=channel, author=upload.user) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): theme_data = parsed_data.get('theme', {}) generate_static_theme_preview(theme_data, version.pk) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}' .format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version
def from_upload(cls, upload, addon, platforms, channel, source=None, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of platform ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ assert parsed_data is not None from olympia.addons.models import AddonFeatureCompatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version( channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id version = cls.objects.create( addon=addon, version=parsed_data['version'], license_id=license_id, source=source, channel=channel, ) log.info( 'New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) # Update the add-on e10s compatibility since we're creating a new # version that may change that. e10s_compatibility = parsed_data.get('e10s_compatibility') if e10s_compatibility is not None: feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0] ) feature_compatibility.update(e10s=e10s_compatibility) compatible_apps = {} for app in parsed_data.get('apps', []): compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version._compatible_apps = compatible_apps if addon.type in [amo.ADDON_SEARCH, amo.ADDON_STATICTHEME]: # Search extensions and static themes are always for all platforms. platforms = [amo.PLATFORM_ALL.id] else: platforms = cls._make_safe_platform_files(platforms) # Create as many files as we have platforms. Update the all_files # cached property on the Version while we're at it, because we might # need it afterwards. version.all_files = [ File.from_upload( upload, version, platform, parsed_data=parsed_data) for platform in platforms] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) version_uploaded.send(sender=version) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): dst_root = os.path.join(user_media_path('addons'), str(addon.id)) theme_data = parsed_data.get('theme', {}) version_root = os.path.join(dst_root, unicode(version.id)) utils.extract_header_img( version.all_files[0].file_path, theme_data, version_root) preview = VersionPreview.objects.create(version=version) generate_static_theme_preview( theme_data, version_root, preview.pk) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}' .format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version
@property def is_user_disabled(self): return (self.file.status == amo.STATUS_DISABLED and self.file.original_status != amo.STATUS_NULL) @is_user_disabled.setter def is_user_disabled(self, disable): # User wants to disable (and the File isn't already). if disable: activity.log_create(amo.LOG.DISABLE_VERSION, self.addon, self) if (file_ := self.file) and file_.status != amo.STATUS_DISABLED: file_.update(original_status=file_.status, status=amo.STATUS_DISABLED) # User wants to re-enable (and user did the disable, not Mozilla). else: activity.log_create(amo.LOG.ENABLE_VERSION, self.addon, self) if (file_ := self.file) and file_.original_status != amo.STATUS_NULL: file_.update(status=file_.original_status, original_status=amo.STATUS_NULL) @cached_property def all_activity(self): # prefetch_related() and not select_related() the ActivityLog to make # sure its transformer is called. return self.versionlog_set.prefetch_related('activity_log').order_by( 'created') @property def compatible_apps(self): # Dicts and search providers don't have compatibility info.