def index_addons(self, addons=None):
"""Index add-ons belonging to that collection."""
from olympia.addons.tasks import index_addons
addon_ids = [addon.id for addon in (addons or self.addons.all())]
if addon_ids:
index_addons.delay(addon_ids)
def update_featured_status(sender, instance, **kwargs):
from olympia.addons.tasks import index_addons
addons = kwargs.get(
'addons', [addon.id for addon in instance.addons.all()])
if addons:
clear_get_featured_ids_cache(None, None)
index_addons.delay(addons)
def generate_static_theme_preview(theme_manifest, version_pk):
# Make sure we import `index_addons` late in the game to avoid having
# a "copy" of it here that won't get mocked by our ESTestCase
from olympia.addons.tasks import index_addons
tmpl = loader.get_template(
'devhub/addons/includes/static_theme_preview_svg.xml')
file_ = File.objects.filter(version_id=version_pk).first()
if not file_:
return
complete_context = _build_static_theme_preview_context(
theme_manifest, file_)
renderings = sorted(amo.THEME_PREVIEW_RENDERINGS.values(),
key=operator.itemgetter('position'))
colors = None
for rendering in renderings:
# Create a Preview for this rendering.
preview = VersionPreview.objects.create(
version_id=version_pk,
position=rendering['position'],
sizes={
'image_format': rendering['image_format'],
'thumbnail_format': rendering['thumbnail_format'],
},
)
# Add the size to the context and render
complete_context.update(svg_render_size=rendering['full'])
if rendering['image_format'] == 'svg':
render_success = render_to_svg(
tmpl,
{
**complete_context,
**TRANSPARENT_UI
},
preview,
rendering['thumbnail'],
theme_manifest,
)
else:
render_success = render_to_png(tmpl, complete_context, preview,
rendering['thumbnail'])
if render_success:
# Extract colors once and store it for all previews.
# Use the thumbnail for extra speed, we don't need to be super accurate.
if colors is None:
colors = extract_colors_from_image(preview.thumbnail_path)
data = {
'sizes': {
'image': rendering['full'],
'thumbnail': rendering['thumbnail'],
'image_format': rendering['image_format'],
'thumbnail_format': rendering['thumbnail_format'],
},
'colors': colors,
}
preview.update(**data)
addon_id = Version.objects.values_list('addon_id',
flat=True).get(id=version_pk)
index_addons.delay([addon_id])
def ban_and_disable_related_content_bulk(cls, users, move_files=False):
"""Admin method to ban users and disable the content they produced.
Similar to deletion, except that the content produced by the user is
forcibly disabled instead of being deleted where possible, and the user
is not fully anonymized: we keep their fxa_id and email so that they
are never able to log back in.
"""
from olympia.addons.models import Addon, AddonUser
from olympia.addons.tasks import index_addons
from olympia.bandwagon.models import Collection
from olympia.files.models import File
from olympia.ratings.models import Rating
# collect affected addons
addon_ids = set(
Addon.unfiltered.exclude(status=amo.STATUS_DELETED).filter(
addonuser__user__in=users).values_list('id', flat=True))
# First addons who have other authors we aren't banning
addon_joint_ids = set(
AddonUser.objects.filter(addon_id__in=addon_ids).exclude(
user__in=users).values_list('addon_id', flat=True))
AddonUser.objects.filter(user__in=users,
addon_id__in=addon_joint_ids).delete()
# Then deal with users who are the sole author
addons_sole = Addon.unfiltered.filter(id__in=addon_ids -
addon_joint_ids)
# set the status to disabled - using the manager update() method
addons_sole.update(status=amo.STATUS_DISABLED)
# collect Files that need to be disabled now the addons are disabled
files_to_disable = File.objects.filter(version__addon__in=addons_sole)
files_to_disable.update(status=amo.STATUS_DISABLED)
if move_files:
# if necessary move the files out of the CDN (expensive operation)
for file_ in files_to_disable:
file_.hide_disabled_file()
# Finally run Addon.force_disable to add the logging; update versions
# Status was already DISABLED so shouldn't fire watch_disabled again.
for addon in addons_sole:
addon.force_disable()
# Don't pass a set to a .delay - sets can't be serialized as JSON
index_addons.delay(list(addon_ids - addon_joint_ids))
# delete the other content associated with the user
Collection.objects.filter(author__in=users).delete()
Rating.objects.filter(user__in=users).delete(
user_responsible=core.get_user())
# And then delete the users.
for user in users:
log.info(f'User ({user}: <{user.email}>) is being '
'anonymized and banned.')
user.banned = user.modified = datetime.now()
user.deleted = True
cls.anonymize_users(users)
cls.objects.bulk_update(users,
fields=('banned', 'deleted', 'modified') +
cls.ANONYMIZED_FIELDS)
def watch_changes(old_attr=None,
new_attr=None,
instance=None,
sender=None,
**kw):
if old_attr is None:
old_attr = {}
if new_attr is None:
new_attr = {}
changes = {
x
for x in new_attr
if not x.startswith('_') and new_attr[x] != old_attr.get(x)
}
# Log email changes.
if 'email' in changes and new_attr['email'] is not None:
log.debug('Creating user history for user: %s' % instance.pk)
UserHistory.objects.create(email=old_attr.get('email'),
user_id=instance.pk)
# If username or display_name changes, reindex the user add-ons, if there
# are any.
if 'username' in changes or 'display_name' in changes:
from olympia.addons.tasks import index_addons
ids = [addon.pk for addon in instance.get_addons_listed()]
if ids:
index_addons.delay(ids)
basket_relevant_changes = ('deleted', 'display_name', 'email', 'homepage',
'last_login', 'location')
if any(field in changes for field in basket_relevant_changes):
from olympia.amo.tasks import sync_object_to_basket
log.info('Triggering a sync of %s %s with basket because of %s change',
'userprofile', instance.pk, 'attribute')
sync_object_to_basket.delay('userprofile', instance.pk)
def update_featured_status(sender, instance, **kwargs):
from olympia.addons.tasks import index_addons
addons = kwargs.get('addons',
[addon.id for addon in instance.addons.all()])
if addons:
clear_get_featured_ids_cache(None, None)
index_addons.delay(addons)
def generate_static_theme_preview(theme_manifest, version_pk):
tmpl = loader.get_template(
'devhub/addons/includes/static_theme_preview_svg.xml')
file_ = File.objects.filter(version_id=version_pk).first()
if not file_:
return
context = _build_static_theme_preview_context(theme_manifest, file_)
sizes = sorted(amo.THEME_PREVIEW_SIZES.values(),
lambda x, y: x['position'] - y['position'])
for size in sizes:
# Create a Preview for this size.
preview = VersionPreview.objects.create(version_id=version_pk,
position=size['position'])
# Add the size to the context and render
context.update(svg_render_size=size['full'])
svg = tmpl.render(context).encode('utf-8')
if write_svg_to_png(svg, preview.image_path):
resize_image(preview.image_path, preview.thumbnail_path,
size['thumbnail'])
pngcrush_image(preview.image_path)
preview_sizes = {}
preview_sizes['image'] = size['full']
preview_sizes['thumbnail'] = size['thumbnail']
preview.update(sizes=preview_sizes)
addon_id = Version.objects.values_list('addon_id',
flat=True).get(id=version_pk)
index_addons.delay([addon_id])
def generate_static_theme_preview(theme_manifest, version_pk):
tmpl = loader.get_template(
'devhub/addons/includes/static_theme_preview_svg.xml')
file_ = File.objects.filter(version_id=version_pk).first()
if not file_:
return
context = _build_static_theme_preview_context(theme_manifest, file_)
sizes = sorted(
amo.THEME_PREVIEW_SIZES.values(),
lambda x, y: x['position'] - y['position'])
for size in sizes:
# Create a Preview for this size.
preview = VersionPreview.objects.create(
version_id=version_pk, position=size['position'])
# Add the size to the context and render
context.update(svg_render_size=size['full'])
svg = tmpl.render(context).encode('utf-8')
if write_svg_to_png(svg, preview.image_path):
resize_image(
preview.image_path, preview.thumbnail_path, size['thumbnail'])
pngcrush_image(preview.image_path)
preview_sizes = {}
preview_sizes['image'] = size['full']
preview_sizes['thumbnail'] = size['thumbnail']
preview.update(sizes=preview_sizes)
addon_id = Version.objects.values_list(
'addon_id', flat=True).get(id=version_pk)
index_addons.delay([addon_id])
def make_recommended_all_apps(apps, schema_editor):
PromotedAddon = apps.get_model('promoted', 'PromotedAddon')
qs = PromotedAddon.objects.filter(group_id=RECOMMENDED.id,
application_id=amo.FIREFOX.id)
for promo in qs:
promo.application_id = None
promo.save()
if qs.count():
index_addons.delay([promoted.addon_id for promoted in qs])
def delete_armagaddon_ratings_for_addons(ids, **kw):
ratings = Rating.objects.filter(addon__in=ids,
**get_armagaddon_ratings_filters())
task_user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
for rating in ratings:
# Normally, deletions are a special kind of save (because we
# soft-delete) and that'd send a post save signal, which would trigger
# addon ratings aggregate computation and reindexing. We specifically
# avoid sending post_save to do those things only once per add-on.
rating.delete(user_responsible=task_user, send_post_save_signal=False)
addon_rating_aggregates(ids)
index_addons.delay(ids)
def delete_armagaddon_ratings_for_addons(ids, **kw):
ratings = Rating.objects.filter(
addon__in=ids, **get_armagaddon_ratings_filters())
task_user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
for rating in ratings:
# Normally, deletions are a special kind of save (because we
# soft-delete) and that'd send a post save signal, which would trigger
# addon ratings aggregate computation and reindexing. We specifically
# avoid sending post_save to do those things only once per add-on.
rating.delete(user_responsible=task_user,
send_post_save_signal=False)
addon_rating_aggregates(ids)
index_addons.delay(ids)
def generate_static_theme_preview(theme_manifest, version_pk):
# Make sure we import `index_addons` late in the game to avoid having
# a "copy" of it here that won't get mocked by our ESTestCase
from olympia.addons.tasks import index_addons
tmpl = loader.get_template(
'devhub/addons/includes/static_theme_preview_svg.xml')
file_ = File.objects.filter(version_id=version_pk).first()
if not file_:
return
context = _build_static_theme_preview_context(theme_manifest, file_)
renderings = sorted(amo.THEME_PREVIEW_RENDERINGS.values(),
key=operator.itemgetter('position'))
colors = None
for rendering in renderings:
# Create a Preview for this size.
preview = VersionPreview.objects.create(
version_id=version_pk,
position=rendering['position'],
sizes={'thumbnail_format': rendering['thumbnail_format']},
)
# Add the size to the context and render
context.update(svg_render_size=rendering['full'])
svg = tmpl.render(context).encode('utf-8')
if write_svg_to_png(svg, preview.image_path):
resize_image(
preview.image_path,
preview.thumbnail_path,
rendering['thumbnail'],
format=rendering['thumbnail_format'],
quality=
35, # It's ignored for png format, so it's fine to always set.
)
pngcrush_image(preview.image_path)
# Extract colors once and store it for all previews.
# Use the thumbnail for extra speed, we don't need to be super accurate.
if colors is None:
colors = extract_colors_from_image(preview.thumbnail_path)
data = {
'sizes': {
'image': rendering['full'],
'thumbnail': rendering['thumbnail'],
'thumbnail_format': rendering['thumbnail_format'],
},
'colors': colors,
}
preview.update(**data)
addon_id = Version.objects.values_list('addon_id',
flat=True).get(id=version_pk)
index_addons.delay([addon_id])
def delete_list_theme_previews(addon_ids, **kw):
# Make sure we import `index_addons` late in the game to avoid having
# a "copy" of it here that won't get mocked by our ESTestCase
from olympia.addons.tasks import index_addons
log.info(
'[%s@%s] Deleting preview sizes for themes starting at id: %s...' %
(len(addon_ids), delete_list_theme_previews.rate_limit, addon_ids[0]))
for addon_id in addon_ids:
log.info('Deleting "list" size previews for theme: %s' % addon_id)
VersionPreview.objects.filter(version__addon_id=addon_id,
sizes__image=[760, 92]).delete()
index_addons.delay(addon_ids)
def set_compatible_apps(self, apps):
from olympia.addons.tasks import index_addons # circular import
# clear any removed applications
self.apps.exclude(application__in=(app.id for app in apps)).delete()
# then save the instances
for applications_versions in apps.values():
if not applications_versions.id:
# set version if we have it, for new instances
applications_versions.version = self
applications_versions.save()
# Update cache on the model.
self.compatible_apps = apps
# Make sure the add-on is properly re-indexed
index_addons.delay([self.addon.id])
def post_delete(sender, instance, **kwargs):
from olympia.addons.tasks import index_addons
if kwargs.get('raw'):
return
if instance.collection.listed:
activity.log_create(amo.LOG.REMOVE_FROM_COLLECTION, instance.addon,
instance.collection)
kwargs['addons'] = [instance.addon]
Collection.post_save(sender, instance.collection, **kwargs)
if instance.collection.id == settings.COLLECTION_FEATURED_THEMES_ID:
# That collection is special: each add-on in it is considered
# recommended, so we need to index the add-on we just removed from
# it.
index_addons.delay([instance.addon.id])
def contributions_url_to_from_www(apps, schema_editor):
Addon = apps.get_model('addons', 'Addon')
to_index = []
for addon in Addon.unfiltered.exclude(
Q(contributions__isnull=True, contributions='')):
try:
new_url = fix_contributions_url(addon.contributions)
if new_url != addon.contributions:
addon.contributions = new_url
addon.save()
to_index.append(addon.id)
except ValueError:
pass
if to_index:
index_addons.delay(to_index)
def ban_and_disable_related_content_bulk(cls, users, move_files=False):
"""Like ban_and_disable_related_content, but in bulk. """
from olympia.addons.models import Addon, AddonUser
from olympia.addons.tasks import index_addons
from olympia.bandwagon.models import Collection
from olympia.files.models import File
from olympia.ratings.models import Rating
# collect affected addons
addon_ids = set(
Addon.unfiltered.exclude(status=amo.STATUS_DELETED).filter(
addonuser__user__in=users).values_list('id', flat=True))
# First addons who have other authors we aren't banning
addon_joint_ids = set(
AddonUser.objects.filter(addon_id__in=addon_ids).exclude(
user__in=users).values_list('addon_id', flat=True))
AddonUser.objects.filter(user__in=users,
addon_id__in=addon_joint_ids).delete()
# Then deal with users who are the sole author
addons_sole = Addon.unfiltered.filter(id__in=addon_ids -
addon_joint_ids)
# set the status to disabled - using the manager update() method
addons_sole.update(status=amo.STATUS_DISABLED)
# collect Files that need to be disabled now the addons are disabled
files_to_disable = File.objects.filter(version__addon__in=addons_sole)
files_to_disable.update(status=amo.STATUS_DISABLED)
if move_files:
# if necessary move the files out of the CDN (expensive operation)
for file_ in files_to_disable:
file_.hide_disabled_file()
# Finally run Addon.force_disable to add the logging; update versions
# Status was already DISABLED so shouldn't fire watch_disabled again.
for addon in addons_sole:
addon.force_disable()
# Don't pass a set to a .delay - sets can't be serialized as JSON
index_addons.delay(list(addon_ids - addon_joint_ids))
# delete the other content associated with the user
Collection.objects.filter(author__in=users).delete()
Rating.objects.filter(user__in=users).delete(
user_responsible=core.get_user())
# And then delete the users.
for user in users:
user.delete(ban_user=True)
def ban_and_disable_related_content_bulk(cls, users, move_files=False):
"""Like ban_and_disable_related_content, but in bulk. """
from olympia.addons.models import Addon, AddonUser
from olympia.addons.tasks import index_addons
from olympia.bandwagon.models import Collection
from olympia.files.models import File
from olympia.ratings.models import Rating
# collect affected addons
addon_ids = set(
Addon.unfiltered.exclude(status=amo.STATUS_DELETED)
.filter(addonuser__user__in=users).values_list('id', flat=True))
# First addons who have other authors we aren't banning
addon_joint_ids = set(
AddonUser.objects.filter(addon_id__in=addon_ids)
.exclude(user__in=users).values_list('addon_id', flat=True))
AddonUser.objects.filter(
user__in=users, addon_id__in=addon_joint_ids).delete()
# Then deal with users who are the sole author
addons_sole = Addon.unfiltered.filter(
id__in=addon_ids - addon_joint_ids)
# set the status to disabled - using the manager update() method
addons_sole.update(status=amo.STATUS_DISABLED)
# collect Files that need to be disabled now the addons are disabled
files_to_disable = File.objects.filter(version__addon__in=addons_sole)
files_to_disable.update(status=amo.STATUS_DISABLED)
if move_files:
# if necessary move the files out of the CDN (expensive operation)
for file_ in files_to_disable:
file_.hide_disabled_file()
# Finally run Addon.force_disable to add the logging; update versions
# Status was already DISABLED so shouldn't fire watch_disabled again.
for addon in addons_sole:
addon.force_disable()
index_addons.delay(addon_ids - addon_joint_ids)
# delete the other content associated with the user
Collection.objects.filter(author__in=users).delete()
Rating.objects.filter(user__in=users).delete(
user_responsible=core.get_user())
# And then delete the users.
for user in users:
user.delete(keep_fxa_id_and_email=True)
def watch_changes(old_attr=None, new_attr=None, instance=None,
sender=None, **kw):
if old_attr is None:
old_attr = {}
if new_attr is None:
new_attr = {}
# Log email changes.
new_email, old_email = new_attr.get('email'), old_attr.get('email')
if old_email and new_email != old_email:
log.debug('Creating user history for user: %s' % instance.pk)
UserHistory.objects.create(email=old_email, user_id=instance.pk)
# If username or display_name changes, reindex the user add-ons, if there
# are any.
if (new_attr.get('username') != old_attr.get('username') or
new_attr.get('display_name') != old_attr.get('display_name')):
from olympia.addons.tasks import index_addons
ids = [addon.pk for addon in instance.get_addons_listed()]
if ids:
index_addons.delay(ids)
def watch_changes(old_attr=None, new_attr=None, instance=None,
sender=None, **kw):
if old_attr is None:
old_attr = {}
if new_attr is None:
new_attr = {}
# Log email changes.
new_email, old_email = new_attr.get('email'), old_attr.get('email')
if old_email and new_email != old_email:
log.debug('Creating user history for user: %s' % instance.pk)
UserHistory.objects.create(email=old_email, user_id=instance.pk)
# If username or display_name changes, reindex the user add-ons, if there
# are any.
if (new_attr.get('username') != old_attr.get('username') or
new_attr.get('display_name') != old_attr.get('display_name')):
from olympia.addons.tasks import index_addons
ids = [addon.pk for addon in instance.get_addons_listed()]
if ids:
index_addons.delay(ids)
def post_save(sender, instance, **kwargs):
"""Update Collection.addon_count and reindex add-on if the collection
is featured."""
from olympia.addons.tasks import index_addons
if kwargs.get('raw'):
return
if instance.collection.listed:
activity.log_create(amo.LOG.ADD_TO_COLLECTION, instance.addon,
instance.collection)
kwargs['addons'] = [instance.addon]
Collection.post_save(sender, instance.collection, **kwargs)
if instance.collection.id == settings.COLLECTION_FEATURED_THEMES_ID:
# That collection is special: each add-on in it is considered
# recommended, so we need to index the corresponding add-on.
# (Note: we are considering the add-on in a given CollectionAddon
# never changes, to change add-ons belonging to a collection we
# add or remove CollectionAddon instances, we never modify the
# addon foreignkey of an existing instance).
index_addons.delay([instance.addon.id])
def generate_static_theme_preview(theme_manifest, version_pk):
# Make sure we import `index_addons` late in the game to avoid having
# a "copy" of it here that won't get mocked by our ESTestCase
from olympia.addons.tasks import index_addons
tmpl = loader.get_template(
'devhub/addons/includes/static_theme_preview_svg.xml')
file_ = File.objects.filter(version_id=version_pk).first()
if not file_:
return
context = _build_static_theme_preview_context(theme_manifest, file_)
sizes = sorted(
amo.THEME_PREVIEW_SIZES.values(), key=operator.itemgetter('position'))
colors = None
for size in sizes:
# Create a Preview for this size.
preview = VersionPreview.objects.create(
version_id=version_pk, position=size['position'])
# Add the size to the context and render
context.update(svg_render_size=size['full'])
svg = tmpl.render(context).encode('utf-8')
if write_svg_to_png(svg, preview.image_path):
resize_image(
preview.image_path, preview.thumbnail_path, size['thumbnail'])
pngcrush_image(preview.image_path)
# Extract colors once and store it for all previews.
# Use the thumbnail for extra speed, we don't need to be super
# accurate.
if colors is None:
colors = extract_colors_from_image(preview.thumbnail_path)
data = {
'sizes': {
'image': size['full'],
'thumbnail': size['thumbnail'],
},
'colors': colors,
}
preview.update(**data)
addon_id = Version.objects.values_list(
'addon_id', flat=True).get(id=version_pk)
index_addons.delay([addon_id])
def generate_static_theme_preview(theme_manifest, version_pk):
tmpl = loader.get_template(
'devhub/addons/includes/static_theme_preview_svg.xml')
file_ = File.objects.filter(version_id=version_pk).first()
if not file_:
return
context = _build_static_theme_preview_context(theme_manifest, file_)
sizes = sorted(amo.THEME_PREVIEW_SIZES.values(),
lambda x, y: x['position'] - y['position'])
colors = None
for size in sizes:
# Create a Preview for this size.
preview = VersionPreview.objects.create(version_id=version_pk,
position=size['position'])
# Add the size to the context and render
context.update(svg_render_size=size['full'])
svg = tmpl.render(context).encode('utf-8')
if write_svg_to_png(svg, preview.image_path):
resize_image(preview.image_path, preview.thumbnail_path,
size['thumbnail'])
pngcrush_image(preview.image_path)
# Extract colors once and store it for all previews.
# Use the thumbnail for extra speed, we don't need to be super
# accurate.
if colors is None:
colors = extract_colors_from_image(preview.thumbnail_path)
data = {
'sizes': {
'image': size['full'],
'thumbnail': size['thumbnail'],
},
'colors': colors,
}
preview.update(**data)
addon_id = Version.objects.values_list('addon_id',
flat=True).get(id=version_pk)
index_addons.delay([addon_id])
