def index_addons(self, addons=None): """Index add-ons belonging to that collection.""" from olympia.addons.tasks import index_addons addon_ids = [addon.id for addon in (addons or self.addons.all())] if addon_ids: index_addons.delay(addon_ids)
def update_featured_status(sender, instance, **kwargs): from olympia.addons.tasks import index_addons addons = kwargs.get( 'addons', [addon.id for addon in instance.addons.all()]) if addons: clear_get_featured_ids_cache(None, None) index_addons.delay(addons)
def generate_static_theme_preview(theme_manifest, version_pk): # Make sure we import `index_addons` late in the game to avoid having # a "copy" of it here that won't get mocked by our ESTestCase from olympia.addons.tasks import index_addons tmpl = loader.get_template( 'devhub/addons/includes/static_theme_preview_svg.xml') file_ = File.objects.filter(version_id=version_pk).first() if not file_: return complete_context = _build_static_theme_preview_context( theme_manifest, file_) renderings = sorted(amo.THEME_PREVIEW_RENDERINGS.values(), key=operator.itemgetter('position')) colors = None for rendering in renderings: # Create a Preview for this rendering. preview = VersionPreview.objects.create( version_id=version_pk, position=rendering['position'], sizes={ 'image_format': rendering['image_format'], 'thumbnail_format': rendering['thumbnail_format'], }, ) # Add the size to the context and render complete_context.update(svg_render_size=rendering['full']) if rendering['image_format'] == 'svg': render_success = render_to_svg( tmpl, { **complete_context, **TRANSPARENT_UI }, preview, rendering['thumbnail'], theme_manifest, ) else: render_success = render_to_png(tmpl, complete_context, preview, rendering['thumbnail']) if render_success: # Extract colors once and store it for all previews. # Use the thumbnail for extra speed, we don't need to be super accurate. if colors is None: colors = extract_colors_from_image(preview.thumbnail_path) data = { 'sizes': { 'image': rendering['full'], 'thumbnail': rendering['thumbnail'], 'image_format': rendering['image_format'], 'thumbnail_format': rendering['thumbnail_format'], }, 'colors': colors, } preview.update(**data) addon_id = Version.objects.values_list('addon_id', flat=True).get(id=version_pk) index_addons.delay([addon_id])
def ban_and_disable_related_content_bulk(cls, users, move_files=False): """Admin method to ban users and disable the content they produced. Similar to deletion, except that the content produced by the user is forcibly disabled instead of being deleted where possible, and the user is not fully anonymized: we keep their fxa_id and email so that they are never able to log back in. """ from olympia.addons.models import Addon, AddonUser from olympia.addons.tasks import index_addons from olympia.bandwagon.models import Collection from olympia.files.models import File from olympia.ratings.models import Rating # collect affected addons addon_ids = set( Addon.unfiltered.exclude(status=amo.STATUS_DELETED).filter( addonuser__user__in=users).values_list('id', flat=True)) # First addons who have other authors we aren't banning addon_joint_ids = set( AddonUser.objects.filter(addon_id__in=addon_ids).exclude( user__in=users).values_list('addon_id', flat=True)) AddonUser.objects.filter(user__in=users, addon_id__in=addon_joint_ids).delete() # Then deal with users who are the sole author addons_sole = Addon.unfiltered.filter(id__in=addon_ids - addon_joint_ids) # set the status to disabled - using the manager update() method addons_sole.update(status=amo.STATUS_DISABLED) # collect Files that need to be disabled now the addons are disabled files_to_disable = File.objects.filter(version__addon__in=addons_sole) files_to_disable.update(status=amo.STATUS_DISABLED) if move_files: # if necessary move the files out of the CDN (expensive operation) for file_ in files_to_disable: file_.hide_disabled_file() # Finally run Addon.force_disable to add the logging; update versions # Status was already DISABLED so shouldn't fire watch_disabled again. for addon in addons_sole: addon.force_disable() # Don't pass a set to a .delay - sets can't be serialized as JSON index_addons.delay(list(addon_ids - addon_joint_ids)) # delete the other content associated with the user Collection.objects.filter(author__in=users).delete() Rating.objects.filter(user__in=users).delete( user_responsible=core.get_user()) # And then delete the users. for user in users: log.info(f'User ({user}: <{user.email}>) is being ' 'anonymized and banned.') user.banned = user.modified = datetime.now() user.deleted = True cls.anonymize_users(users) cls.objects.bulk_update(users, fields=('banned', 'deleted', 'modified') + cls.ANONYMIZED_FIELDS)
def watch_changes(old_attr=None, new_attr=None, instance=None, sender=None, **kw): if old_attr is None: old_attr = {} if new_attr is None: new_attr = {} changes = { x for x in new_attr if not x.startswith('_') and new_attr[x] != old_attr.get(x) } # Log email changes. if 'email' in changes and new_attr['email'] is not None: log.debug('Creating user history for user: %s' % instance.pk) UserHistory.objects.create(email=old_attr.get('email'), user_id=instance.pk) # If username or display_name changes, reindex the user add-ons, if there # are any. if 'username' in changes or 'display_name' in changes: from olympia.addons.tasks import index_addons ids = [addon.pk for addon in instance.get_addons_listed()] if ids: index_addons.delay(ids) basket_relevant_changes = ('deleted', 'display_name', 'email', 'homepage', 'last_login', 'location') if any(field in changes for field in basket_relevant_changes): from olympia.amo.tasks import sync_object_to_basket log.info('Triggering a sync of %s %s with basket because of %s change', 'userprofile', instance.pk, 'attribute') sync_object_to_basket.delay('userprofile', instance.pk)
def update_featured_status(sender, instance, **kwargs): from olympia.addons.tasks import index_addons addons = kwargs.get('addons', [addon.id for addon in instance.addons.all()]) if addons: clear_get_featured_ids_cache(None, None) index_addons.delay(addons)
def generate_static_theme_preview(theme_manifest, version_pk): tmpl = loader.get_template( 'devhub/addons/includes/static_theme_preview_svg.xml') file_ = File.objects.filter(version_id=version_pk).first() if not file_: return context = _build_static_theme_preview_context(theme_manifest, file_) sizes = sorted(amo.THEME_PREVIEW_SIZES.values(), lambda x, y: x['position'] - y['position']) for size in sizes: # Create a Preview for this size. preview = VersionPreview.objects.create(version_id=version_pk, position=size['position']) # Add the size to the context and render context.update(svg_render_size=size['full']) svg = tmpl.render(context).encode('utf-8') if write_svg_to_png(svg, preview.image_path): resize_image(preview.image_path, preview.thumbnail_path, size['thumbnail']) pngcrush_image(preview.image_path) preview_sizes = {} preview_sizes['image'] = size['full'] preview_sizes['thumbnail'] = size['thumbnail'] preview.update(sizes=preview_sizes) addon_id = Version.objects.values_list('addon_id', flat=True).get(id=version_pk) index_addons.delay([addon_id])
def generate_static_theme_preview(theme_manifest, version_pk): tmpl = loader.get_template( 'devhub/addons/includes/static_theme_preview_svg.xml') file_ = File.objects.filter(version_id=version_pk).first() if not file_: return context = _build_static_theme_preview_context(theme_manifest, file_) sizes = sorted( amo.THEME_PREVIEW_SIZES.values(), lambda x, y: x['position'] - y['position']) for size in sizes: # Create a Preview for this size. preview = VersionPreview.objects.create( version_id=version_pk, position=size['position']) # Add the size to the context and render context.update(svg_render_size=size['full']) svg = tmpl.render(context).encode('utf-8') if write_svg_to_png(svg, preview.image_path): resize_image( preview.image_path, preview.thumbnail_path, size['thumbnail']) pngcrush_image(preview.image_path) preview_sizes = {} preview_sizes['image'] = size['full'] preview_sizes['thumbnail'] = size['thumbnail'] preview.update(sizes=preview_sizes) addon_id = Version.objects.values_list( 'addon_id', flat=True).get(id=version_pk) index_addons.delay([addon_id])
def make_recommended_all_apps(apps, schema_editor): PromotedAddon = apps.get_model('promoted', 'PromotedAddon') qs = PromotedAddon.objects.filter(group_id=RECOMMENDED.id, application_id=amo.FIREFOX.id) for promo in qs: promo.application_id = None promo.save() if qs.count(): index_addons.delay([promoted.addon_id for promoted in qs])
def delete_armagaddon_ratings_for_addons(ids, **kw): ratings = Rating.objects.filter(addon__in=ids, **get_armagaddon_ratings_filters()) task_user = UserProfile.objects.get(pk=settings.TASK_USER_ID) for rating in ratings: # Normally, deletions are a special kind of save (because we # soft-delete) and that'd send a post save signal, which would trigger # addon ratings aggregate computation and reindexing. We specifically # avoid sending post_save to do those things only once per add-on. rating.delete(user_responsible=task_user, send_post_save_signal=False) addon_rating_aggregates(ids) index_addons.delay(ids)
def delete_armagaddon_ratings_for_addons(ids, **kw): ratings = Rating.objects.filter( addon__in=ids, **get_armagaddon_ratings_filters()) task_user = UserProfile.objects.get(pk=settings.TASK_USER_ID) for rating in ratings: # Normally, deletions are a special kind of save (because we # soft-delete) and that'd send a post save signal, which would trigger # addon ratings aggregate computation and reindexing. We specifically # avoid sending post_save to do those things only once per add-on. rating.delete(user_responsible=task_user, send_post_save_signal=False) addon_rating_aggregates(ids) index_addons.delay(ids)
def generate_static_theme_preview(theme_manifest, version_pk): # Make sure we import `index_addons` late in the game to avoid having # a "copy" of it here that won't get mocked by our ESTestCase from olympia.addons.tasks import index_addons tmpl = loader.get_template( 'devhub/addons/includes/static_theme_preview_svg.xml') file_ = File.objects.filter(version_id=version_pk).first() if not file_: return context = _build_static_theme_preview_context(theme_manifest, file_) renderings = sorted(amo.THEME_PREVIEW_RENDERINGS.values(), key=operator.itemgetter('position')) colors = None for rendering in renderings: # Create a Preview for this size. preview = VersionPreview.objects.create( version_id=version_pk, position=rendering['position'], sizes={'thumbnail_format': rendering['thumbnail_format']}, ) # Add the size to the context and render context.update(svg_render_size=rendering['full']) svg = tmpl.render(context).encode('utf-8') if write_svg_to_png(svg, preview.image_path): resize_image( preview.image_path, preview.thumbnail_path, rendering['thumbnail'], format=rendering['thumbnail_format'], quality= 35, # It's ignored for png format, so it's fine to always set. ) pngcrush_image(preview.image_path) # Extract colors once and store it for all previews. # Use the thumbnail for extra speed, we don't need to be super accurate. if colors is None: colors = extract_colors_from_image(preview.thumbnail_path) data = { 'sizes': { 'image': rendering['full'], 'thumbnail': rendering['thumbnail'], 'thumbnail_format': rendering['thumbnail_format'], }, 'colors': colors, } preview.update(**data) addon_id = Version.objects.values_list('addon_id', flat=True).get(id=version_pk) index_addons.delay([addon_id])
def delete_list_theme_previews(addon_ids, **kw): # Make sure we import `index_addons` late in the game to avoid having # a "copy" of it here that won't get mocked by our ESTestCase from olympia.addons.tasks import index_addons log.info( '[%s@%s] Deleting preview sizes for themes starting at id: %s...' % (len(addon_ids), delete_list_theme_previews.rate_limit, addon_ids[0])) for addon_id in addon_ids: log.info('Deleting "list" size previews for theme: %s' % addon_id) VersionPreview.objects.filter(version__addon_id=addon_id, sizes__image=[760, 92]).delete() index_addons.delay(addon_ids)
def set_compatible_apps(self, apps): from olympia.addons.tasks import index_addons # circular import # clear any removed applications self.apps.exclude(application__in=(app.id for app in apps)).delete() # then save the instances for applications_versions in apps.values(): if not applications_versions.id: # set version if we have it, for new instances applications_versions.version = self applications_versions.save() # Update cache on the model. self.compatible_apps = apps # Make sure the add-on is properly re-indexed index_addons.delay([self.addon.id])
def post_delete(sender, instance, **kwargs): from olympia.addons.tasks import index_addons if kwargs.get('raw'): return if instance.collection.listed: activity.log_create(amo.LOG.REMOVE_FROM_COLLECTION, instance.addon, instance.collection) kwargs['addons'] = [instance.addon] Collection.post_save(sender, instance.collection, **kwargs) if instance.collection.id == settings.COLLECTION_FEATURED_THEMES_ID: # That collection is special: each add-on in it is considered # recommended, so we need to index the add-on we just removed from # it. index_addons.delay([instance.addon.id])
def contributions_url_to_from_www(apps, schema_editor): Addon = apps.get_model('addons', 'Addon') to_index = [] for addon in Addon.unfiltered.exclude( Q(contributions__isnull=True, contributions='')): try: new_url = fix_contributions_url(addon.contributions) if new_url != addon.contributions: addon.contributions = new_url addon.save() to_index.append(addon.id) except ValueError: pass if to_index: index_addons.delay(to_index)
def ban_and_disable_related_content_bulk(cls, users, move_files=False): """Like ban_and_disable_related_content, but in bulk. """ from olympia.addons.models import Addon, AddonUser from olympia.addons.tasks import index_addons from olympia.bandwagon.models import Collection from olympia.files.models import File from olympia.ratings.models import Rating # collect affected addons addon_ids = set( Addon.unfiltered.exclude(status=amo.STATUS_DELETED).filter( addonuser__user__in=users).values_list('id', flat=True)) # First addons who have other authors we aren't banning addon_joint_ids = set( AddonUser.objects.filter(addon_id__in=addon_ids).exclude( user__in=users).values_list('addon_id', flat=True)) AddonUser.objects.filter(user__in=users, addon_id__in=addon_joint_ids).delete() # Then deal with users who are the sole author addons_sole = Addon.unfiltered.filter(id__in=addon_ids - addon_joint_ids) # set the status to disabled - using the manager update() method addons_sole.update(status=amo.STATUS_DISABLED) # collect Files that need to be disabled now the addons are disabled files_to_disable = File.objects.filter(version__addon__in=addons_sole) files_to_disable.update(status=amo.STATUS_DISABLED) if move_files: # if necessary move the files out of the CDN (expensive operation) for file_ in files_to_disable: file_.hide_disabled_file() # Finally run Addon.force_disable to add the logging; update versions # Status was already DISABLED so shouldn't fire watch_disabled again. for addon in addons_sole: addon.force_disable() # Don't pass a set to a .delay - sets can't be serialized as JSON index_addons.delay(list(addon_ids - addon_joint_ids)) # delete the other content associated with the user Collection.objects.filter(author__in=users).delete() Rating.objects.filter(user__in=users).delete( user_responsible=core.get_user()) # And then delete the users. for user in users: user.delete(ban_user=True)
def ban_and_disable_related_content_bulk(cls, users, move_files=False): """Like ban_and_disable_related_content, but in bulk. """ from olympia.addons.models import Addon, AddonUser from olympia.addons.tasks import index_addons from olympia.bandwagon.models import Collection from olympia.files.models import File from olympia.ratings.models import Rating # collect affected addons addon_ids = set( Addon.unfiltered.exclude(status=amo.STATUS_DELETED) .filter(addonuser__user__in=users).values_list('id', flat=True)) # First addons who have other authors we aren't banning addon_joint_ids = set( AddonUser.objects.filter(addon_id__in=addon_ids) .exclude(user__in=users).values_list('addon_id', flat=True)) AddonUser.objects.filter( user__in=users, addon_id__in=addon_joint_ids).delete() # Then deal with users who are the sole author addons_sole = Addon.unfiltered.filter( id__in=addon_ids - addon_joint_ids) # set the status to disabled - using the manager update() method addons_sole.update(status=amo.STATUS_DISABLED) # collect Files that need to be disabled now the addons are disabled files_to_disable = File.objects.filter(version__addon__in=addons_sole) files_to_disable.update(status=amo.STATUS_DISABLED) if move_files: # if necessary move the files out of the CDN (expensive operation) for file_ in files_to_disable: file_.hide_disabled_file() # Finally run Addon.force_disable to add the logging; update versions # Status was already DISABLED so shouldn't fire watch_disabled again. for addon in addons_sole: addon.force_disable() index_addons.delay(addon_ids - addon_joint_ids) # delete the other content associated with the user Collection.objects.filter(author__in=users).delete() Rating.objects.filter(user__in=users).delete( user_responsible=core.get_user()) # And then delete the users. for user in users: user.delete(keep_fxa_id_and_email=True)
def watch_changes(old_attr=None, new_attr=None, instance=None, sender=None, **kw): if old_attr is None: old_attr = {} if new_attr is None: new_attr = {} # Log email changes. new_email, old_email = new_attr.get('email'), old_attr.get('email') if old_email and new_email != old_email: log.debug('Creating user history for user: %s' % instance.pk) UserHistory.objects.create(email=old_email, user_id=instance.pk) # If username or display_name changes, reindex the user add-ons, if there # are any. if (new_attr.get('username') != old_attr.get('username') or new_attr.get('display_name') != old_attr.get('display_name')): from olympia.addons.tasks import index_addons ids = [addon.pk for addon in instance.get_addons_listed()] if ids: index_addons.delay(ids)
def watch_changes(old_attr=None, new_attr=None, instance=None, sender=None, **kw): if old_attr is None: old_attr = {} if new_attr is None: new_attr = {} # Log email changes. new_email, old_email = new_attr.get('email'), old_attr.get('email') if old_email and new_email != old_email: log.debug('Creating user history for user: %s' % instance.pk) UserHistory.objects.create(email=old_email, user_id=instance.pk) # If username or display_name changes, reindex the user add-ons, if there # are any. if (new_attr.get('username') != old_attr.get('username') or new_attr.get('display_name') != old_attr.get('display_name')): from olympia.addons.tasks import index_addons ids = [addon.pk for addon in instance.get_addons_listed()] if ids: index_addons.delay(ids)
def post_save(sender, instance, **kwargs): """Update Collection.addon_count and reindex add-on if the collection is featured.""" from olympia.addons.tasks import index_addons if kwargs.get('raw'): return if instance.collection.listed: activity.log_create(amo.LOG.ADD_TO_COLLECTION, instance.addon, instance.collection) kwargs['addons'] = [instance.addon] Collection.post_save(sender, instance.collection, **kwargs) if instance.collection.id == settings.COLLECTION_FEATURED_THEMES_ID: # That collection is special: each add-on in it is considered # recommended, so we need to index the corresponding add-on. # (Note: we are considering the add-on in a given CollectionAddon # never changes, to change add-ons belonging to a collection we # add or remove CollectionAddon instances, we never modify the # addon foreignkey of an existing instance). index_addons.delay([instance.addon.id])
def generate_static_theme_preview(theme_manifest, version_pk): # Make sure we import `index_addons` late in the game to avoid having # a "copy" of it here that won't get mocked by our ESTestCase from olympia.addons.tasks import index_addons tmpl = loader.get_template( 'devhub/addons/includes/static_theme_preview_svg.xml') file_ = File.objects.filter(version_id=version_pk).first() if not file_: return context = _build_static_theme_preview_context(theme_manifest, file_) sizes = sorted( amo.THEME_PREVIEW_SIZES.values(), key=operator.itemgetter('position')) colors = None for size in sizes: # Create a Preview for this size. preview = VersionPreview.objects.create( version_id=version_pk, position=size['position']) # Add the size to the context and render context.update(svg_render_size=size['full']) svg = tmpl.render(context).encode('utf-8') if write_svg_to_png(svg, preview.image_path): resize_image( preview.image_path, preview.thumbnail_path, size['thumbnail']) pngcrush_image(preview.image_path) # Extract colors once and store it for all previews. # Use the thumbnail for extra speed, we don't need to be super # accurate. if colors is None: colors = extract_colors_from_image(preview.thumbnail_path) data = { 'sizes': { 'image': size['full'], 'thumbnail': size['thumbnail'], }, 'colors': colors, } preview.update(**data) addon_id = Version.objects.values_list( 'addon_id', flat=True).get(id=version_pk) index_addons.delay([addon_id])
def generate_static_theme_preview(theme_manifest, version_pk): tmpl = loader.get_template( 'devhub/addons/includes/static_theme_preview_svg.xml') file_ = File.objects.filter(version_id=version_pk).first() if not file_: return context = _build_static_theme_preview_context(theme_manifest, file_) sizes = sorted(amo.THEME_PREVIEW_SIZES.values(), lambda x, y: x['position'] - y['position']) colors = None for size in sizes: # Create a Preview for this size. preview = VersionPreview.objects.create(version_id=version_pk, position=size['position']) # Add the size to the context and render context.update(svg_render_size=size['full']) svg = tmpl.render(context).encode('utf-8') if write_svg_to_png(svg, preview.image_path): resize_image(preview.image_path, preview.thumbnail_path, size['thumbnail']) pngcrush_image(preview.image_path) # Extract colors once and store it for all previews. # Use the thumbnail for extra speed, we don't need to be super # accurate. if colors is None: colors = extract_colors_from_image(preview.thumbnail_path) data = { 'sizes': { 'image': size['full'], 'thumbnail': size['thumbnail'], }, 'colors': colors, } preview.update(**data) addon_id = Version.objects.values_list('addon_id', flat=True).get(id=version_pk) index_addons.delay([addon_id])