def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) attrs = ('created', 'default_language', 'id', 'modified', 'slug', 'status', 'version') doc = dict(zip(attrs, attrgetter(*attrs)(obj))) # is_disabled is here for compatibility with other filters, but never # set to True at the moment, and not present in the model. doc['is_disabled'] = False doc['name_sort'] = unicode(obj.name).lower() doc['guid'] = unicode(obj.uuid) # Handle localized fields. This adds both the field used for search and # the one with all translations for the API. for field in cls.translated_fields: doc.update(cls.extract_field_translations( obj, field, include_field_for_search=True)) # Handle language-specific analyzers. for field in cls.fields_with_language_analyzers: doc.update(cls.extract_field_analyzed_translations(obj, field)) return doc
def test_multiple_objects_with_multiple_translations(self): obj = FancyModel.objects.create() obj.purified = { 'fr': 'French Purified', 'en-us': 'English Purified' } obj.save() obj2 = FancyModel.objects.create(purified='English 2 Linkified') obj2.linkified = { 'fr': 'French 2 Linkified', 'en-us': 'English 2 Linkified', 'es': 'Spanish 2 Linkified' } obj2.save() attach_trans_dict(FancyModel, [obj, obj2]) eq_(set(obj.translations[obj.purified_id]), set([('en-us', 'English Purified'), ('fr', 'French Purified')])) eq_(set(obj2.translations[obj2.linkified_id]), set([('en-us', 'English 2 Linkified'), ('es', 'Spanish 2 Linkified'), ('fr', 'French 2 Linkified')]))
def test_basic(self): obj = self.FancyModel.objects.create( purified='Purified <script>alert(42)</script>!', linkified='Linkified <script>alert(42)</script>!') # Quick sanity checks: is description properly escaped? The underlying # implementation should leave localized_string un-escaped but never use # it for __unicode__. We depend on this behaviour later in the test. ok_('<script>' in obj.purified.localized_string) ok_('<script>' not in obj.purified.localized_string_clean) ok_('<script>' in obj.linkified.localized_string) ok_('<script>' not in obj.linkified.localized_string_clean) # Attach trans dict. attach_trans_dict(self.FancyModel, [obj]) ok_(isinstance(obj.translations, collections.defaultdict)) translations = dict(obj.translations) # addon.translations is a defaultdict. eq_(obj.translations['whatever'], []) # No-translated fields should be absent. ok_(None not in translations) # Build expected translations dict. expected_translations = { obj.purified_id: [('en-us', unicode(obj.purified))], obj.linkified_id: [('en-us', unicode(obj.linkified))], } eq_(translations, expected_translations)
def test_basic(self): obj = FancyModel.objects.create( purified='Purified <script>alert(42)</script>!', linkified='Linkified <script>alert(42)</script>!') # Quick sanity checks: is description properly escaped? The underlying # implementation should leave localized_string un-escaped but never use # it for __unicode__. We depend on this behaviour later in the test. ok_('<script>' in obj.purified.localized_string) ok_('<script>' not in obj.purified.localized_string_clean) ok_('<script>' in obj.linkified.localized_string) ok_('<script>' not in obj.linkified.localized_string_clean) # Attach trans dict. attach_trans_dict(FancyModel, [obj]) ok_(isinstance(obj.translations, collections.defaultdict)) translations = dict(obj.translations) # addon.translations is a defaultdict. eq_(obj.translations['whatever'], []) # No-translated fields should be absent. ok_(None not in translations) # Build expected translations dict. expected_translations = { obj.purified_id: [ ('en-us', unicode(obj.purified))], obj.linkified_id: [ ('en-us', unicode(obj.linkified))], } eq_(translations, expected_translations)
def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) attach_trans_dict(Webapp, [obj.app]) doc = { 'id': obj.id, 'app': obj.app_id, 'background_color': obj.background_color, 'created': obj.created, 'image_hash': obj.image_hash, 'item_type': feed.FEED_TYPE_APP, 'preview': {'id': obj.preview.id, 'thumbnail_size': obj.preview.thumbnail_size, 'thumbnail_url': obj.preview.thumbnail_url} if getattr(obj, 'preview') else None, 'pullquote_attribution': obj.pullquote_attribution, 'pullquote_rating': obj.pullquote_rating, 'search_names': list( set(string for _, string in obj.app.translations[obj.app.name_id])), 'slug': obj.slug, 'type': obj.type, } # Handle localized fields. for field in ('description', 'pullquote_text'): doc.update(format_translation_es(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): if obj is None: obj = cls.get_model().get(pk=pk) attach_trans_dict(cls.get_model(), [obj]) doc = { 'id': obj.id, 'apps': list(obj.apps().values_list('id', flat=True)), 'carrier': mkt.carriers.CARRIER_CHOICE_DICT[obj.carrier].slug, 'created': obj.created, 'image_hash': obj.image_hash, 'image_landing_hash': obj.image_landing_hash, 'item_type': feed.FEED_TYPE_SHELF, 'region': mkt.regions.REGIONS_CHOICES_ID_DICT[obj.region].slug, 'search_names': list(set(string for _, string in obj.translations[obj.name_id])), 'slug': obj.slug, } # Handle localized fields. for field in ('description', 'name'): doc.update(format_translation_es(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) attrs = ('created', 'default_locale', 'id', 'icon_hash', 'icon_type', 'is_disabled', 'last_updated', 'modified', 'status') doc = dict(zip(attrs, attrgetter(*attrs)(obj))) doc['category'] = obj.categories or [] doc['device'] = obj.devices or [] doc['title_sort'] = unicode(obj.title).lower() doc['region_exclusions'] = obj.region_exclusions or [] # Add boost, popularity, trending values. doc.update(cls.extract_popularity_trending_boost(obj)) # Handle localized fields. This adds both the field used for search and # the one with all translations for the API. for field in cls.translated_fields: doc.update(cls.extract_field_translations( obj, field, include_field_for_search=True)) # Handle language-specific analyzers. for field in cls.fields_with_language_analyzers: doc.update(cls.extract_field_analyzed_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): from mkt.feed.models import FeedCollectionMembership if obj is None: obj = cls.get_model().objects.get(pk=pk) attach_trans_dict(cls.get_model(), [obj]) doc = { 'id': obj.id, 'apps': list(obj.apps().values_list('id', flat=True)), 'background_color': obj.background_color, 'color': obj.color, 'created': obj.created, 'group_apps': {}, # Map of app IDs to index in group_names below. 'group_names': [], # List of ES-serialized group names. 'image_hash': obj.image_hash, 'item_type': feed.FEED_TYPE_COLL, 'search_names': list(set(string for _, string in obj.translations[obj.name_id])), 'slug': obj.slug, 'type': obj.type, } # Grouped apps. Key off of translation, pointed to app IDs. memberships = obj.feedcollectionmembership_set.all() attach_trans_dict(FeedCollectionMembership, memberships) for member in memberships: if member.group: group_translation = cls.extract_field_translations( member, 'group') if group_translation not in doc['group_names']: doc['group_names'].append(group_translation) doc['group_apps'][member.app_id] = ( doc['group_names'].index(group_translation)) # Handle localized fields. for field in ('description', 'name'): doc.update(cls.extract_field_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): from mkt.feed.models import FeedShelfMembership if obj is None: obj = cls.get_model().get(pk=pk) attach_trans_dict(cls.get_model(), [obj]) doc = { 'id': obj.id, 'apps': list(obj.apps().values_list('id', flat=True)), 'carrier': mkt.carriers.CARRIER_CHOICE_DICT[obj.carrier].slug, 'created': obj.created, 'group_apps': {}, # Map of app IDs to index in group_names below. 'group_names': [], # List of ES-serialized group names. 'image_hash': obj.image_hash, 'image_landing_hash': obj.image_landing_hash, 'item_type': feed.FEED_TYPE_SHELF, 'region': mkt.regions.REGIONS_CHOICES_ID_DICT[obj.region].slug, 'search_names': list(set(string for _, string in obj.translations[obj.name_id])), 'slug': obj.slug, } # Grouped apps. Key off of translation, pointed to app IDs. memberships = obj.feedshelfmembership_set.all() attach_trans_dict(FeedShelfMembership, memberships) for member in memberships: if member.group: group_translation = cls.extract_field_translations( member, 'group') if group_translation not in doc['group_names']: doc['group_names'].append(group_translation) doc['group_apps'][member.app_id] = ( doc['group_names'].index(group_translation)) # Handle localized fields. for field in ('description', 'name'): doc.update(cls.extract_field_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) attrs = ('author', 'created', 'default_language', 'icon_hash', 'id', 'last_updated', 'modified', 'slug', 'status') doc = dict(zip(attrs, attrgetter(*attrs)(obj))) doc['device'] = obj.devices doc['guid'] = unicode(obj.uuid) doc['is_deleted'] = obj.deleted doc['is_disabled'] = obj.disabled if obj.status == STATUS_PUBLIC: doc['latest_public_version'] = { 'id': obj.latest_public_version.pk, 'created': obj.latest_public_version.created, 'size': obj.latest_public_version.size, 'version': obj.latest_public_version.version } else: doc['latest_public_version'] = None doc['name_sort'] = unicode(obj.name).lower() # Find the first reviewed date (used in sort). doc['reviewed'] = obj.versions.public().aggregate( Min('reviewed')).get('reviewed__min') # Add boost, popularity, trending values. doc.update(cls.extract_popularity_trending_boost(obj)) # Handle localized fields. This adds both the field used for search and # the one with all translations for the API. for field in cls.translated_fields: doc.update( cls.extract_field_translations(obj, field, include_field_for_search=True)) # Handle language-specific analyzers. for field in cls.fields_with_language_analyzers: doc.update(cls.extract_field_analyzed_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) # Attach tags (keywords). attach_tags([obj]) attrs = ('created', 'default_locale', 'id', 'icon_hash', 'icon_type', 'is_disabled', 'last_updated', 'mobile_url', 'modified', 'promo_img_hash', 'status', 'tv_url', 'url') doc = dict(zip(attrs, attrgetter(*attrs)(obj))) doc['category'] = obj.categories or [] doc['device'] = obj.devices or [] doc['name_sort'] = unicode(obj.name).lower() doc['preferred_regions'] = obj.preferred_regions or [] doc['tags'] = getattr(obj, 'keywords_list', []) doc['tv_featured'] = obj.keywords.filter( tag_text='featured-tv').exists() doc['url_tokenized'] = cls.strip_url(obj.url) # For now, websites are not reviewed, since we're manually injecting # data, so just use last_updated. doc['reviewed'] = obj.last_updated # Add boost, popularity, trending values. doc.update(cls.extract_popularity_trending_boost(obj)) # Handle localized fields. This adds both the field used for search and # the one with all translations for the API. for field in cls.translated_fields: doc.update( cls.extract_field_translations(obj, field, include_field_for_search=True)) # Handle language-specific analyzers. for field in cls.fields_with_language_analyzers: doc.update(cls.extract_field_analyzed_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) # Attach tags (keywords). attach_tags([obj]) attrs = ('created', 'default_locale', 'id', 'icon_hash', 'icon_type', 'is_disabled', 'last_updated', 'mobile_url', 'modified', 'promo_img_hash', 'status', 'tv_url', 'url') doc = dict(zip(attrs, attrgetter(*attrs)(obj))) doc['category'] = obj.categories or [] doc['device'] = obj.devices or [] doc['developer_name'] = (unicode(obj.developer_name) if obj.developer_name else None) doc['name_sort'] = unicode(obj.name).lower() doc['preferred_regions'] = obj.preferred_regions or [] doc['tags'] = getattr(obj, 'keywords_list', []) doc['tv_featured'] = obj.keywords.filter( tag_text='featured-tv').exists() doc['url_tokenized'] = cls.strip_url(obj.url) # For now, websites are not reviewed, since we're manually injecting # data, so just use last_updated. doc['reviewed'] = obj.last_updated # Add boost, popularity, trending values. doc.update(cls.extract_popularity_trending_boost(obj)) # Handle localized fields. This adds both the field used for search and # the one with all translations for the API. for field in cls.translated_fields: doc.update(cls.extract_field_translations( obj, field, include_field_for_search=True)) # Handle language-specific analyzers. for field in cls.fields_with_language_analyzers: doc.update(cls.extract_field_analyzed_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) attrs = ('author', 'created', 'default_language', 'icon_hash', 'id', 'last_updated', 'modified', 'slug', 'status') doc = dict(zip(attrs, attrgetter(*attrs)(obj))) doc['device'] = obj.devices doc['guid'] = unicode(obj.uuid) doc['is_deleted'] = obj.deleted doc['is_disabled'] = obj.disabled if obj.status == STATUS_PUBLIC: doc['latest_public_version'] = { 'id': obj.latest_public_version.pk, 'created': obj.latest_public_version.created, 'size': obj.latest_public_version.size, 'version': obj.latest_public_version.version } else: doc['latest_public_version'] = None doc['name_sort'] = unicode(obj.name).lower() # Find the first reviewed date (used in sort). doc['reviewed'] = obj.versions.public().aggregate( Min('reviewed')).get('reviewed__min') # Add boost, popularity, trending values. doc.update(cls.extract_popularity_trending_boost(obj)) # Handle localized fields. This adds both the field used for search and # the one with all translations for the API. for field in cls.translated_fields: doc.update(cls.extract_field_translations( obj, field, include_field_for_search=True)) # Handle language-specific analyzers. for field in cls.fields_with_language_analyzers: doc.update(cls.extract_field_analyzed_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): from mkt.feed.models import FeedCollectionMembership if obj is None: obj = cls.get_model().objects.get(pk=pk) attach_trans_dict(cls.get_model(), [obj]) doc = { 'id': obj.id, 'apps': list(obj.apps().values_list('id', flat=True)), 'background_color': obj.background_color, 'color': obj.color, 'created': obj.created, 'group_apps': {}, # Map of app IDs to index in group_names below. 'group_names': [], # List of ES-serialized group names. 'image_hash': obj.image_hash, 'item_type': feed.FEED_TYPE_COLL, 'search_names': list( set(string for _, string in obj.translations[obj.name_id])), 'slug': obj.slug, 'type': obj.type, } # Grouped apps. Key off of translation, pointed to app IDs. memberships = obj.feedcollectionmembership_set.all() attach_trans_dict(FeedCollectionMembership, memberships) for member in memberships: if member.group: group_translation = cls.extract_field_translations(member, 'group') if group_translation not in doc['group_names']: doc['group_names'].append(group_translation) doc['group_apps'][member.app_id] = ( doc['group_names'].index(group_translation)) # Handle localized fields. for field in ('description', 'name'): doc.update(cls.extract_field_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): from mkt.feed.models import FeedShelfMembership if obj is None: obj = cls.get_model().get(pk=pk) attach_trans_dict(cls.get_model(), [obj]) doc = { 'id': obj.id, 'apps': list(obj.apps().values_list('id', flat=True)), 'carrier': mkt.carriers.CARRIER_CHOICE_DICT[obj.carrier].slug, 'created': obj.created, 'group_apps': {}, # Map of app IDs to index in group_names below. 'group_names': [], # List of ES-serialized group names. 'image_hash': obj.image_hash, 'image_landing_hash': obj.image_landing_hash, 'item_type': feed.FEED_TYPE_SHELF, 'region': mkt.regions.REGIONS_CHOICES_ID_DICT[obj.region].slug, 'search_names': list(set(string for _, string in obj.translations[obj.name_id])), 'slug': obj.slug, } # Grouped apps. Key off of translation, pointed to app IDs. memberships = obj.feedshelfmembership_set.all() attach_trans_dict(FeedShelfMembership, memberships) for member in memberships: if member.group: group_translation = cls.extract_field_translations(member, 'group') if group_translation not in doc['group_names']: doc['group_names'].append(group_translation) doc['group_apps'][member.app_id] = ( doc['group_names'].index(group_translation)) # Handle localized fields. for field in ('description', 'name'): doc.update(cls.extract_field_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): from mkt.feed.models import FeedCollectionMembership if obj is None: obj = cls.get_model().objects.get(pk=pk) attach_trans_dict(cls.get_model(), [obj]) doc = { "id": obj.id, "apps": list(obj.apps().values_list("id", flat=True)), "background_color": obj.background_color, "color": obj.color, "created": obj.created, "group_apps": {}, # Map of app IDs to index in group_names below. "group_names": [], # List of ES-serialized group names. "image_hash": obj.image_hash, "item_type": feed.FEED_TYPE_COLL, "search_names": list(set(string for _, string in obj.translations[obj.name_id])), "slug": obj.slug, "type": obj.type, } # Grouped apps. Key off of translation, pointed to app IDs. memberships = obj.feedcollectionmembership_set.all() attach_trans_dict(FeedCollectionMembership, memberships) for member in memberships: if member.group: group_translation = cls.extract_field_translations(member, "group") if group_translation not in doc["group_names"]: doc["group_names"].append(group_translation) doc["group_apps"][member.app_id] = doc["group_names"].index(group_translation) # Handle localized fields. for field in ("description", "name"): doc.update(cls.extract_field_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): from mkt.feed.models import FeedShelfMembership if obj is None: obj = cls.get_model().get(pk=pk) attach_trans_dict(cls.get_model(), [obj]) doc = { "id": obj.id, "apps": list(obj.apps().values_list("id", flat=True)), "carrier": mkt.carriers.CARRIER_CHOICE_DICT[obj.carrier].slug, "created": obj.created, "group_apps": {}, # Map of app IDs to index in group_names below. "group_names": [], # List of ES-serialized group names. "image_hash": obj.image_hash, "image_landing_hash": obj.image_landing_hash, "item_type": feed.FEED_TYPE_SHELF, "region": mkt.regions.REGIONS_CHOICES_ID_DICT[obj.region].slug, "search_names": list(set(string for _, string in obj.translations[obj.name_id])), "slug": obj.slug, } # Grouped apps. Key off of translation, pointed to app IDs. memberships = obj.feedshelfmembership_set.all() attach_trans_dict(FeedShelfMembership, memberships) for member in memberships: if member.group: group_translation = cls.extract_field_translations(member, "group") if group_translation not in doc["group_names"]: doc["group_names"].append(group_translation) doc["group_apps"][member.app_id] = doc["group_names"].index(group_translation) # Handle localized fields. for field in ("description", "name"): doc.update(cls.extract_field_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): """Converts this instance into an Elasticsearch document""" if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach translations for searching and indexing. attach_trans_dict(cls.get_model(), [obj]) attach_trans_dict(Webapp, [obj.app]) doc = { "id": obj.id, "app": obj.app_id, "background_color": obj.background_color, "color": obj.color, "created": obj.created, "image_hash": obj.image_hash, "item_type": feed.FEED_TYPE_APP, "preview": { "id": obj.preview.id, "thumbnail_size": obj.preview.thumbnail_size, "thumbnail_url": obj.preview.thumbnail_url, } if getattr(obj, "preview") else None, "pullquote_attribution": obj.pullquote_attribution, "pullquote_rating": obj.pullquote_rating, "search_names": list(set(string for _, string in obj.app.translations[obj.app.name_id])), "slug": obj.slug, "type": obj.type, } # Handle localized fields. for field in ("description", "pullquote_text"): doc.update(cls.extract_field_translations(obj, field)) return doc
def extract_document(cls, pk=None, obj=None): """Extracts the ElasticSearch index document for this instance.""" from mkt.webapps.models import (AppFeatures, attach_devices, attach_prices, attach_translations, RatingDescriptors, RatingInteractives) if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach everything we need to index apps. for transform in (attach_devices, attach_prices, attach_tags, attach_translations): transform([obj]) latest_version = obj.latest_version version = obj.current_version geodata = obj.geodata features = (version.features.to_dict() if version else AppFeatures().to_dict()) try: status = latest_version.statuses[0][1] if latest_version else None except IndexError: status = None attrs = ('app_slug', 'bayesian_rating', 'created', 'default_locale', 'guid', 'icon_hash', 'id', 'is_disabled', 'is_offline', 'file_size', 'last_updated', 'modified', 'premium_type', 'promo_img_hash', 'status', 'uses_flash') d = dict(zip(attrs, attrgetter(*attrs)(obj))) d['app_type'] = obj.app_type_id d['author'] = obj.developer_name d['banner_regions'] = geodata.banner_regions_slugs() d['category'] = obj.categories if obj.categories else [] d['content_ratings'] = (obj.get_content_ratings_by_body(es=True) or None) try: d['content_descriptors'] = obj.rating_descriptors.to_keys() except RatingDescriptors.DoesNotExist: d['content_descriptors'] = [] d['current_version'] = version.version if version else None d['device'] = getattr(obj, 'device_ids', []) d['features'] = features d['has_public_stats'] = obj.public_stats try: d['interactive_elements'] = obj.rating_interactives.to_keys() except RatingInteractives.DoesNotExist: d['interactive_elements'] = [] d['installs_allowed_from'] = ( version.manifest.get('installs_allowed_from', ['*']) if version else ['*']) d['is_priority'] = obj.priority_review is_escalated = obj.escalationqueue_set.exists() d['is_escalated'] = is_escalated d['escalation_date'] = (obj.escalationqueue_set.get().created if is_escalated else None) is_rereviewed = obj.rereviewqueue_set.exists() d['is_rereviewed'] = is_rereviewed d['rereview_date'] = (obj.rereviewqueue_set.get().created if is_rereviewed else None) if latest_version: d['latest_version'] = { 'status': status, 'is_privileged': latest_version.is_privileged, 'has_editor_comment': latest_version.has_editor_comment, 'has_info_request': latest_version.has_info_request, 'nomination_date': latest_version.nomination, 'created_date': latest_version.created, } else: d['latest_version'] = { 'status': None, 'is_privileged': None, 'has_editor_comment': None, 'has_info_request': None, 'nomination_date': None, 'created_date': None, } d['manifest_url'] = obj.get_manifest_url() d['package_path'] = obj.get_package_path() d['name_sort'] = unicode(obj.name).lower() d['owners'] = [au.user.id for au in obj.addonuser_set.filter(role=mkt.AUTHOR_ROLE_OWNER)] d['previews'] = [{'filetype': p.filetype, 'modified': p.modified, 'id': p.id, 'sizes': p.sizes} for p in obj.previews.all()] try: p = obj.addonpremium.price d['price_tier'] = p.name except AddonPremium.DoesNotExist: d['price_tier'] = None d['ratings'] = { 'average': obj.average_rating, 'count': obj.total_reviews, } d['region_exclusions'] = obj.get_excluded_region_ids() d['reviewed'] = obj.versions.filter( deleted=False).aggregate(Min('reviewed')).get('reviewed__min') # The default locale of the app is considered "supported" by default. supported_locales = [obj.default_locale] other_locales = (filter(None, version.supported_locales.split(',')) if version else []) if other_locales: supported_locales.extend(other_locales) d['supported_locales'] = list(set(supported_locales)) d['tags'] = getattr(obj, 'tags_list', []) if obj.upsell and obj.upsell.premium.is_published(): upsell_obj = obj.upsell.premium d['upsell'] = { 'id': upsell_obj.id, 'app_slug': upsell_obj.app_slug, 'icon_url': upsell_obj.get_icon_url(128), # TODO: Store all localizations of upsell.name. 'name': unicode(upsell_obj.name), 'region_exclusions': upsell_obj.get_excluded_region_ids() } d['versions'] = [dict(version=v.version, resource_uri=reverse_version(v)) for v in obj.versions.all()] # Handle localized fields. # This adds both the field used for search and the one with # all translations for the API. for field in ('description', 'name'): d.update(cls.extract_field_translations( obj, field, include_field_for_search=True)) # This adds only the field with all the translations for the API, we # don't need to search on those. for field in ('homepage', 'support_email', 'support_url'): d.update(cls.extract_field_translations(obj, field)) if version: attach_trans_dict(version._meta.model, [version]) d.update(cls.extract_field_translations( version, 'release_notes', db_field='releasenotes_id')) else: d['release_notes_translations'] = None attach_trans_dict(geodata._meta.model, [geodata]) d.update(cls.extract_field_translations(geodata, 'banner_message')) # Add boost, popularity, trending values. d.update(cls.extract_popularity_trending_boost(obj)) # If the app is compatible with Firefox OS, push suggestion data in the # index - This will be used by RocketbarView API, which is specific to # Firefox OS. if DEVICE_GAIA.id in d['device'] and obj.is_published(): d['name_suggest'] = { 'input': d['name'], 'output': unicode(obj.id), # We only care about the payload. 'weight': int(d['boost']), 'payload': { 'default_locale': d['default_locale'], 'icon_hash': d['icon_hash'], 'id': d['id'], 'manifest_url': d['manifest_url'], 'modified': d['modified'], 'name_translations': d['name_translations'], 'slug': d['app_slug'], } } for field in ('name', 'description'): d.update(cls.extract_field_analyzed_translations(obj, field)) return d
def extract_document(cls, pk=None, obj=None): """Extracts the ElasticSearch index document for this instance.""" from mkt.webapps.models import (AppFeatures, attach_devices, attach_prices, attach_tags, attach_translations, Geodata, Installed, RatingDescriptors, RatingInteractives) if obj is None: obj = cls.get_model().objects.no_cache().get(pk=pk) # Attach everything we need to index apps. for transform in (attach_devices, attach_prices, attach_tags, attach_translations): transform([obj]) latest_version = obj.latest_version version = obj.current_version geodata = obj.geodata features = (version.features.to_dict() if version else AppFeatures().to_dict()) try: status = latest_version.statuses[0][1] if latest_version else None except IndexError: status = None installed_ids = list(Installed.objects.filter(addon=obj) .values_list('id', flat=True)) attrs = ('app_slug', 'bayesian_rating', 'created', 'id', 'is_disabled', 'last_updated', 'modified', 'premium_type', 'status', 'uses_flash', 'weekly_downloads') d = dict(zip(attrs, attrgetter(*attrs)(obj))) d['boost'] = len(installed_ids) or 1 d['app_type'] = obj.app_type_id d['author'] = obj.developer_name d['banner_regions'] = geodata.banner_regions_slugs() d['category'] = obj.categories if obj.categories else [] if obj.is_published: d['collection'] = [{'id': cms.collection_id, 'order': cms.order} for cms in obj.collectionmembership_set.all()] else: d['collection'] = [] d['content_ratings'] = (obj.get_content_ratings_by_body(es=True) or None) try: d['content_descriptors'] = obj.rating_descriptors.to_keys() except RatingDescriptors.DoesNotExist: d['content_descriptors'] = [] d['current_version'] = version.version if version else None d['default_locale'] = obj.default_locale d['description'] = list( set(string for _, string in obj.translations[obj.description_id])) d['device'] = getattr(obj, 'device_ids', []) d['features'] = features d['has_public_stats'] = obj.public_stats d['icon_hash'] = obj.icon_hash try: d['interactive_elements'] = obj.rating_interactives.to_keys() except RatingInteractives.DoesNotExist: d['interactive_elements'] = [] d['is_escalated'] = obj.escalationqueue_set.exists() d['is_offline'] = getattr(obj, 'is_offline', False) d['is_priority'] = obj.priority_review d['is_rereviewed'] = obj.rereviewqueue_set.exists() if latest_version: d['latest_version'] = { 'status': status, 'is_privileged': latest_version.is_privileged, 'has_editor_comment': latest_version.has_editor_comment, 'has_info_request': latest_version.has_info_request, 'nomination_date': latest_version.nomination, 'created_date': latest_version.created, } else: d['latest_version'] = { 'status': None, 'is_privileged': None, 'has_editor_comment': None, 'has_info_request': None, 'nomination_date': None, 'created_date': None, } d['manifest_url'] = obj.get_manifest_url() d['package_path'] = obj.get_package_path() d['name'] = list( set(string for _, string in obj.translations[obj.name_id])) d['name_sort'] = unicode(obj.name).lower() d['owners'] = [au.user.id for au in obj.addonuser_set.filter(role=amo.AUTHOR_ROLE_OWNER)] d['popularity'] = len(installed_ids) d['previews'] = [{'filetype': p.filetype, 'modified': p.modified, 'id': p.id, 'sizes': p.sizes} for p in obj.previews.all()] try: p = obj.addonpremium.price d['price_tier'] = p.name except AddonPremium.DoesNotExist: d['price_tier'] = None d['ratings'] = { 'average': obj.average_rating, 'count': obj.total_reviews, } d['region_exclusions'] = obj.get_excluded_region_ids() d['reviewed'] = obj.versions.filter( deleted=False).aggregate(Min('reviewed')).get('reviewed__min') if version: d['supported_locales'] = filter( None, version.supported_locales.split(',')) else: d['supported_locales'] = [] d['tags'] = getattr(obj, 'tag_list', []) if obj.upsell and obj.upsell.premium.is_published(): upsell_obj = obj.upsell.premium d['upsell'] = { 'id': upsell_obj.id, 'app_slug': upsell_obj.app_slug, 'icon_url': upsell_obj.get_icon_url(128), # TODO: Store all localizations of upsell.name. 'name': unicode(upsell_obj.name), 'region_exclusions': upsell_obj.get_excluded_region_ids() } d['versions'] = [dict(version=v.version, resource_uri=reverse_version(v)) for v in obj.versions.all()] # Handle our localized fields. for field in ('description', 'homepage', 'name', 'support_email', 'support_url'): d['%s_translations' % field] = [ {'lang': to_language(lang), 'string': string} for lang, string in obj.translations[getattr(obj, '%s_id' % field)] if string] if version: attach_trans_dict(Version, [version]) d['release_notes_translations'] = [ {'lang': to_language(lang), 'string': string} for lang, string in version.translations[version.releasenotes_id]] else: d['release_notes_translations'] = None attach_trans_dict(Geodata, [geodata]) d['banner_message_translations'] = [ {'lang': to_language(lang), 'string': string} for lang, string in geodata.translations[geodata.banner_message_id]] for region in mkt.regions.ALL_REGION_IDS: d['popularity_%s' % region] = d['popularity'] # Bump the boost if the add-on is public. if obj.status == amo.STATUS_PUBLIC: d['boost'] = max(d['boost'], 1) * 4 # If the app is compatible with Firefox OS, push suggestion data in the # index - This will be used by RocketbarView API, which is specific to # Firefox OS. if DEVICE_GAIA.id in d['device'] and obj.is_published(): d['name_suggest'] = { 'input': d['name'], 'output': unicode(obj.id), # We only care about the payload. 'weight': d['boost'], 'payload': { 'default_locale': d['default_locale'], 'icon_hash': d['icon_hash'], 'id': d['id'], 'manifest_url': d['manifest_url'], 'modified': d['modified'], 'name_translations': d['name_translations'], 'slug': d['app_slug'], } } # Indices for each language. languages is a list of locales we want to # index with analyzer if the string's locale matches. for analyzer, languages in amo.SEARCH_ANALYZER_MAP.iteritems(): if (not settings.ES_USE_PLUGINS and analyzer in amo.SEARCH_ANALYZER_PLUGINS): continue d['name_' + analyzer] = list( set(string for locale, string in obj.translations[obj.name_id] if locale.lower() in languages)) d['description_' + analyzer] = list( set(string for locale, string in obj.translations[obj.description_id] if locale.lower() in languages)) return d
def extract_document(cls, pk=None, obj=None): """Extracts the ElasticSearch index document for this instance.""" from mkt.webapps.models import (AppFeatures, attach_devices, attach_prices, attach_translations, RatingDescriptors, RatingInteractives) if obj is None: obj = cls.get_model().objects.get(pk=pk) # Attach everything we need to index apps. for transform in (attach_devices, attach_prices, attach_tags, attach_translations): transform([obj]) latest_version = obj.latest_version version = obj.current_version features = (version.features.to_dict() if version else AppFeatures().to_dict()) try: status = latest_version.statuses[0][1] if latest_version else None except IndexError: status = None attrs = ('app_slug', 'bayesian_rating', 'created', 'default_locale', 'guid', 'hosted_url', 'icon_hash', 'id', 'is_disabled', 'is_offline', 'file_size', 'last_updated', 'modified', 'premium_type', 'promo_img_hash', 'status') d = dict(zip(attrs, attrgetter(*attrs)(obj))) d['app_type'] = obj.app_type_id d['author'] = obj.developer_name d['category'] = obj.categories if obj.categories else [] d['content_ratings'] = (obj.get_content_ratings_by_body(es=True) or None) try: d['content_descriptors'] = obj.rating_descriptors.to_keys() except RatingDescriptors.DoesNotExist: d['content_descriptors'] = [] d['current_version'] = version.version if version else None d['device'] = getattr(obj, 'device_ids', []) d['features'] = features d['has_public_stats'] = obj.public_stats try: d['interactive_elements'] = obj.rating_interactives.to_keys() except RatingInteractives.DoesNotExist: d['interactive_elements'] = [] d['installs_allowed_from'] = (version.manifest.get( 'installs_allowed_from', ['*']) if version else ['*']) d['is_priority'] = obj.priority_review is_escalated = obj.escalationqueue_set.exists() d['is_escalated'] = is_escalated d['escalation_date'] = (obj.escalationqueue_set.get().created if is_escalated else None) is_rereviewed = obj.rereviewqueue_set.exists() d['is_rereviewed'] = is_rereviewed d['rereview_date'] = (obj.rereviewqueue_set.get().created if is_rereviewed else None) if latest_version: d['latest_version'] = { 'status': status, 'is_privileged': latest_version.is_privileged, 'has_editor_comment': latest_version.has_editor_comment, 'has_info_request': latest_version.has_info_request, 'nomination_date': latest_version.nomination, 'created_date': latest_version.created, } else: d['latest_version'] = { 'status': None, 'is_privileged': None, 'has_editor_comment': None, 'has_info_request': None, 'nomination_date': None, 'created_date': None, } d['manifest_url'] = obj.get_manifest_url() d['package_path'] = obj.get_package_path() d['name_sort'] = unicode(obj.name).lower() d['owners'] = [ au.user.id for au in obj.addonuser_set.filter(role=mkt.AUTHOR_ROLE_OWNER) ] d['previews'] = [{ 'filetype': p.filetype, 'modified': p.modified, 'id': p.id, 'sizes': p.sizes } for p in obj.previews.all()] try: p = obj.addonpremium.price d['price_tier'] = p.name except AddonPremium.DoesNotExist: d['price_tier'] = None d['ratings'] = { 'average': obj.average_rating, 'count': obj.total_reviews, } d['region_exclusions'] = obj.get_excluded_region_ids() d['reviewed'] = obj.versions.filter(deleted=False).aggregate( Min('reviewed')).get('reviewed__min') # The default locale of the app is considered "supported" by default. supported_locales = [obj.default_locale] other_locales = (filter(None, version.supported_locales.split(',')) if version else []) if other_locales: supported_locales.extend(other_locales) d['supported_locales'] = list(set(supported_locales)) d['tags'] = getattr(obj, 'tags_list', []) d['tv_featured'] = obj.tags.filter(tag_text='featured-tv').exists() if obj.upsell and obj.upsell.premium.is_published(): upsell_obj = obj.upsell.premium d['upsell'] = { 'id': upsell_obj.id, 'app_slug': upsell_obj.app_slug, 'icon_url': upsell_obj.get_icon_url(128), # TODO: Store all localizations of upsell.name. 'name': unicode(upsell_obj.name), 'region_exclusions': upsell_obj.get_excluded_region_ids() } d['versions'] = [ dict(version=v.version, resource_uri=reverse_version(v)) for v in obj.versions.all() ] # Handle localized fields. # This adds both the field used for search and the one with # all translations for the API. for field in ('description', 'name'): d.update( cls.extract_field_translations(obj, field, include_field_for_search=True)) # This adds only the field with all the translations for the API, we # don't need to search on those. for field in ('homepage', 'support_email', 'support_url'): d.update(cls.extract_field_translations(obj, field)) if version: attach_trans_dict(version._meta.model, [version]) d.update( cls.extract_field_translations(version, 'release_notes', db_field='releasenotes_id')) else: d['release_notes_translations'] = None # Add boost, popularity, trending values. d.update(cls.extract_popularity_trending_boost(obj)) # If the app is compatible with Firefox OS, push suggestion data in the # index - This will be used by RocketbarView API, which is specific to # Firefox OS. if DEVICE_GAIA.id in d['device'] and obj.is_published(): d['name_suggest'] = { 'input': d['name'], 'output': unicode(obj.id), # We only care about the payload. 'weight': int(d['boost']), 'payload': { 'default_locale': d['default_locale'], 'icon_hash': d['icon_hash'], 'id': d['id'], 'manifest_url': d['manifest_url'], 'modified': d['modified'], 'name_translations': d['name_translations'], 'slug': d['app_slug'], } } for field in ('name', 'description'): d.update(cls.extract_field_analyzed_translations(obj, field)) return d