class DataDeclarationIndex(CelerySearchIndex, indexes.Indexable): def get_model(self): return DataDeclaration def get_updated_field(self): return 'updated' text = indexes.CharField(document=True, use_template=True) pk = indexes.IntegerField(indexed=True, stored=True, faceted=True) cohorts = indexes.MultiValueField(indexed=True, stored=True, faceted=True) consent_status = indexes.CharField(model_attr='consent_status', indexed=True, stored=True, faceted=True) data_types = indexes.MultiValueField(indexed=True, stored=True, faceted=True) data_types_generated = indexes.MultiValueField(indexed=True, stored=True, faceted=True) data_types_received = indexes.MultiValueField(indexed=True, stored=True, faceted=True) deidentification_method = indexes.CharField(indexed=True, stored=True, faceted=True) embargo_date = indexes.DateField(indexed=True, stored=True) end_of_storage_duration = indexes.DateField(indexed=True, stored=True) has_special_subjects = indexes.BooleanField(indexed=True, stored=True, faceted=True) local_custodians = indexes.MultiValueField(indexed=True, stored=True, faceted=True) other_external_id = indexes.CharField(indexed=True, stored=True, faceted=True) project = indexes.CharField(indexed=True, stored=True, faceted=True) special_subjects_description = indexes.CharField(indexed=True, stored=True, faceted=True) subjects_category = indexes.CharField(indexed=True, stored=True, faceted=True) submission_id = indexes.CharField(indexed=True, stored=True, faceted=True) title = indexes.CharField(indexed=True, stored=True, faceted=True) title_l = indexes.CharField(indexed=False, stored=True) unique_id = indexes.CharField(indexed=True, stored=True, faceted=True) autocomplete = indexes.EdgeNgramField() def prepare_autocomplete(self, obj): text_parts = [ obj.title, str(obj.dataset), " ".join([str(c for c in obj.cohorts.all())]), " ".join([str(l) for l in obj.dataset.local_custodians.all()]) ] if obj.dataset.project: text_parts.append(str(obj.dataset.project)) return " ".join(text_parts) def prepare_title(self, obj): return obj.title def prepare_title_l(self, obj): if obj.title: return obj.title.lower().strip() def prepare_deidentification_method(self, obj): return obj.deidentification_method.name def prepare_data_use_restrictions(self, obj): return [o.restriction_class for o in obj.data_use_restrictions.all()] def prepare_subjects_category(self, obj): return obj.subjects_category.name def prepare_cohorts(self, obj): return [str(c) for c in obj.cohorts.all()] def prepare_project(self, obj): if obj.dataset.project: return str(obj.dataset.project) return None def prepare_local_custodians(self, obj): return [u.full_name for u in obj.dataset.local_custodians.all()]
class SolrMockSecretKeySearchIndex(indexes.SearchIndex, indexes.Indexable): Th3S3cr3tK3y = indexes.CharField(document=True, model_attr="author") def get_model(self): return MockModel
class WhooshMockSearchIndex(indexes.SearchIndex): text = indexes.CharField(document=True, use_template=True) name = indexes.CharField(model_attr='author') pub_date = indexes.DateField(model_attr='pub_date')
class GoodsinfoIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) def get_model(self): return Goodsinfo def index_queryset(self, using=None): return self.get_model().objects.all()
class UserIndex(indexes.SearchIndex): text = indexes.CharField(document=True, use_template=True) def index_queryset(self): return User.objects.all()
class CharPKMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, model_attr='key') def get_model(self): return CharPKMockModel
class ProductIndex(CoreProductIndex): # Линзы для очков # astigmatic = indexes.CharField(null=True, faceted=True) # color = indexes.CharField(null=True, faceted=True) # design = indexes.CharField(null=True, faceted=True) # manufacturer = indexes.CharField(null=True, faceted=True) # warranty = indexes.CharField(null=True, faceted=True) brand = indexes.CharField(null=True, faceted=True) cover = indexes.CharField(null=True, faceted=True) index = indexes.DecimalField(null=True, faceted=True) lenses_type = indexes.CharField(null=True, faceted=True) material = indexes.CharField(null=True, faceted=True) properties = indexes.CharField(null=True, faceted=True) purpose = indexes.CharField(null=True, faceted=True) reflex = indexes.CharField(null=True, faceted=True) # def prepare_astigmatic(self, obj): # try: # return obj.attribute_values.get(attribute__code='Astigmatic').value_as_text # except ProductAttributeValue.DoesNotExist: # return None def prepare_brand(self, obj): try: return obj.attribute_values.get( attribute__code='Brand').value_as_text except ProductAttributeValue.DoesNotExist: return None # def prepare_color(self, obj): # try: # return obj.attribute_values.get(attribute__code='Color').value_as_text # except ProductAttributeValue.DoesNotExist: # return None def prepare_cover(self, obj): try: return obj.attribute_values.get( attribute__code='Cover').value_as_text except ProductAttributeValue.DoesNotExist: return None # def prepare_design(self, obj): # try: # return obj.attribute_values.get(attribute__code='Design').value_as_text # except ProductAttributeValue.DoesNotExist: # return None def prepare_index(self, obj): try: return obj.attribute_values.get(attribute__code='Index').value except ProductAttributeValue.DoesNotExist: return None def prepare_lenses_type(self, obj): try: return obj.attribute_values.get( attribute__code='Lenses_type').value_as_text except ProductAttributeValue.DoesNotExist: return None def prepare_manufacturer(self, obj): try: return obj.attribute_values.get( attribute__code='Manufacturer').value_as_text except ProductAttributeValue.DoesNotExist: return None def prepare_material(self, obj): try: return obj.attribute_values.get( attribute__code='Material').value_as_text except ProductAttributeValue.DoesNotExist: return None def prepare_properties(self, obj): try: return obj.attribute_values.get( attribute__code='Properties').value_as_text except ProductAttributeValue.DoesNotExist: return None def prepare_purpose(self, obj): try: return obj.attribute_values.get( attribute__code='Purpose').value_as_text except ProductAttributeValue.DoesNotExist: return None def prepare_reflex(self, obj): try: return obj.attribute_values.get( attribute__code='Reflex').value_as_text except ProductAttributeValue.DoesNotExist: return None
class StoryIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) author = indexes.FacetCharField(model_attr='author') published = indexes.FacetDateTimeField(model_attr='published') created = indexes.FacetDateTimeField(model_attr='created') last_edited = indexes.FacetDateTimeField(model_attr='last_edited') # TODO: Use a meta class to dynamically populate these from "official" # tag sets topic_ids = indexes.FacetMultiValueField() organization_ids = indexes.FacetMultiValueField() project_ids = indexes.FacetMultiValueField() language_ids = indexes.FacetMultiValueField() place_ids = indexes.FacetMultiValueField() points = GeoHashMultiValueField() num_points = indexes.IntegerField() suggestions = TextSpellField() def get_model(self): return Story def prepare_topic_ids(self, obj): return [topic.id for topic in obj.topics.all()] def prepare_organization_ids(self, obj): return [ organization.organization_id for organization in obj.organizations.all() ] def prepare_project_ids(self, obj): return [project.project_id for project in obj.projects.all()] def prepare_language_ids(self, obj): return obj.get_languages() def prepare_place_ids(self, obj): return [place.place_id for place in obj.inherited_places] def prepare_points(self, obj): return ["%s,%s" % (point[0], point[1]) for point in obj.points] def prepare_num_points(self, obj): return len(obj.points) def prepare(self, obj): prepared_data = super(StoryIndex, self).prepare(obj) prepared_data['suggestions'] = prepared_data['text'] return prepared_data def index_queryset(self, using=None): """ Get the default QuerySet to index when doing a full update. Excludes unpublish stories, template stories, and connected stories. """ return Story.objects.filter(status__exact='published', is_template=False)\ .exclude(source__relation_type='connected') def should_update(self, instance, **kwargs): """ Determine if an object should be updated in the index. """ should_update = True translation_set = getattr(instance, instance.translation_set) if translation_set.count() == 0: should_update = False if 'action' in kwargs: # The signal is m2m_changed. We only want to update # on the post actions if kwargs['action'] in ('pre_add', 'pre_remove', 'pre_clear'): should_update = False return should_update def should_remove_on_update(self, instance, **kwargs): if instance.status != 'published': return True if instance.is_template == True: return True if instance.is_connected() == True: return True return False def update_object(self, instance, using=None, **kwargs): """ Update the index for a single object. Attached to the class's post-save hook. This version removes unpublished stories from the index """ if self.should_remove_on_update(instance, **kwargs): self.remove_object(instance, using, **kwargs) else: super(StoryIndex, self).update_object(instance, using, **kwargs) def translation_update_object(self, instance, **kwargs): """Signal handler for updating story index when the translation changes""" # Deal with race condition when stories are deleted # See issue #138 try: self.update_object(instance.story) except Story.DoesNotExist: pass def location_update_object(self, instance, **kwargs): """Signal handler for updating story index when a related location changes""" for story in instance.stories.all(): self.update_object(story) def section_translation_update_object(self, instance, **kwargs): """ Signal handler for updating story index when a related section translation changes This is needed because the section titles in all languages are part of the document field of the index. """ self.update_object(instance.section.story) def asset_translation_update_object(self, instance, **kwargs): """ Signal handler for updating story index when a related text asset translation changes This is needed because the text from text assets is part of the document field in the index. """ stories = [] if instance.asset.type == 'text': for section in instance.asset.sections.all(): # Should I use a set here to make this faster? if section.story not in stories: stories.append(section.story) for story in stories: self.update_object(story) def cache_story_for_delete(self, instance, **kwargs): """ Store a reference to the section asset's story This makes the story available to post_delete signal handlers, because it won't neccessarily be available via instance.section.story at that point. This is designed to be attached to the pre_delete signal """ instance._story = instance.section.story def asset_relation_update_object(self, instance, **kwargs): """ Signal handler for when an asset to section relationship is created or destroyed. This is needed because the text from assets is part of the document field of the index. """ if instance.asset.type == 'text': # Try using the cached story. This will be present if # we're deleting the section asset story = getattr(instance, '_story', None) if story is None: # No cached story present, it's safe to get it by following # the relations story = instance.section.story self.update_object(story)
class AuthorsIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True) name = indexes.CharField(model_attr='name') def get_model(self): return Author
class NoteIndex(CelerySearchIndex, indexes.Indexable): text = indexes.CharField(document=True, model_attr='content') def get_model(self): return Note
class DocumentIndex(indexes.SearchIndex, indexes.Indexable): id = indexes.IntegerField(model_attr='id') abstract = indexes.CharField(model_attr="abstract", boost=1.5) category__gn_description = indexes.CharField( model_attr="category__gn_description", null=True) csw_type = indexes.CharField(model_attr="csw_type") csw_wkt_geometry = indexes.CharField(model_attr="csw_wkt_geometry") detail_url = indexes.CharField(model_attr="get_absolute_url") owner__username = indexes.CharField(model_attr="owner", faceted=True, null=True) popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) srid = indexes.CharField(model_attr="srid") supplemental_information = indexes.CharField( model_attr="supplemental_information", null=True) thumbnail_url = indexes.CharField(model_attr="thumbnail_url", null=True) uuid = indexes.CharField(model_attr="uuid") title = indexes.CharField(model_attr="title", boost=2) date = indexes.DateTimeField(model_attr="date") text = indexes.EdgeNgramField(document=True, use_template=True, stored=False) type = indexes.CharField(faceted=True) title_sortable = indexes.CharField(indexed=False, stored=False) # Necessary for sorting category = indexes.CharField(model_attr="category__identifier", faceted=True, null=True, stored=True) bbox_left = indexes.FloatField(model_attr="bbox_x0", null=True, stored=False) bbox_right = indexes.FloatField(model_attr="bbox_x1", null=True, stored=False) bbox_bottom = indexes.FloatField(model_attr="bbox_y0", null=True, stored=False) bbox_top = indexes.FloatField(model_attr="bbox_y1", null=True, stored=False) temporal_extent_start = indexes.DateTimeField( model_attr="temporal_extent_start", null=True, stored=False) temporal_extent_end = indexes.DateTimeField( model_attr="temporal_extent_end", null=True, stored=False) keywords = indexes.MultiValueField(model_attr="keyword_slug_list", null=True, faceted=True, stored=True) regions = indexes.MultiValueField(model_attr="region_name_list", null=True, faceted=True, stored=True) popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) num_ratings = indexes.IntegerField(stored=False) num_comments = indexes.IntegerField(stored=False) def get_model(self): return Document def prepare_type(self, obj): return "document" def prepare_rating(self, obj): ct = ContentType.objects.get_for_model(obj) try: rating = OverallRating.objects.filter( object_id=obj.pk, content_type=ct).aggregate(r=Avg("rating"))["r"] return float(str(rating or "0")) except OverallRating.DoesNotExist: return 0.0 def prepare_num_ratings(self, obj): ct = ContentType.objects.get_for_model(obj) try: return OverallRating.objects.filter(object_id=obj.pk, content_type=ct).all().count() except OverallRating.DoesNotExist: return 0 def prepare_num_comments(self, obj): try: return Comment.objects.filter( object_id=obj.pk, content_type=ContentType.objects.get_for_model( obj)).all().count() except: return 0 def prepare_title_sortable(self, obj): return obj.title.lower().lstrip()
class OrganizationIndex(indexes.SearchIndex, indexes.Indexable): class Meta: model = Organization name = indexes.CharField(model_attr='name', indexed=True, stored=True) text = indexes.CharField(document=True, use_template=True) slug = indexes.CharField(model_attr='slug', indexed=True, stored=True) cobweb_url = indexes.CharField(model_attr='get_absolute_url', indexed=True, stored=True) short_name = indexes.CharField(model_attr='short_name', null=True, indexed=True, stored=True) full_name = indexes.CharField(model_attr='full_name', null=True, indexed=True, stored=True) administrators = indexes.CharField(indexed=True, null=True, stored=True) address = indexes.CharField(model_attr='address', null=True, indexed=True, stored=True) telephone_number = indexes.CharField(model_attr='telephone_number', null=True, indexed=True, stored=True) url = indexes.CharField(model_attr='url', null=True, indexed=True, stored=True) email_address = indexes.CharField(model_attr='email_address', null=True, indexed=True, stored=True) contact = indexes.CharField(model_attr='contact__get_absolute_url', null=True, indexed=True, stored=True) parent_organization = indexes.CharField( model_attr='parent_organization__get_absolute_url', null=True, indexed=True, stored=True) description = indexes.CharField(model_attr='description', null=True, indexed=True, stored=True) identifier = indexes.CharField(model_attr='identifier', null=True, indexed=True, stored=True) n_claimed = indexes.IntegerField(model_attr='n_claimed', indexed=True, stored=True) n_held = indexes.IntegerField(model_attr='n_held', indexed=True, stored=True) def get_model(self): return Organization def index_queryset(self, using=None): return (self.get_model().objects.all().prefetch_related( 'contact', 'parent_organization')) def prepare_administrators(self, obj: Organization) -> str: return '\n'.join(user.get_absolute_url() for user in obj.administrators.all())
class ResourceIndex(indexes.SearchIndex, indexes.Indexable): class Meta: model = Resource name = indexes.CharField(model_attr='url', stored=True, indexed=False) text = indexes.CharField(document=True, use_template=True) url = indexes.CharField(model_attr='url', stored=True, indexed=True) title = indexes.CharField(null=True, stored=True, indexed=True) # status = indexes.CharField(stored=True, indexed=True) description = indexes.CharField(null=True, stored=True, indexed=True) language = indexes.CharField(null=True, stored=True, indexed=True) tags = indexes.MultiValueField(null=True, stored=True, indexed=True) subject_headings = indexes.MultiValueField(null=True, stored=True, indexed=True) def get_model(self): return Resource def index_queryset(self, using=None): return (self.get_model().objects.all().prefetch_related( 'nominations', 'imported_records')) def prepare_title(self, obj: Resource) -> str: titles = {n.title for n in obj.nominations.all()} for r in obj.imported_records.all(): if 'title' in r.metadata: titles.update(r.metadata['title']) return '\n'.join(x for x in titles if x) def prepare_description(self, obj: Resource) -> str: descriptions = {n.description for n in obj.nominations.all()} for r in obj.imported_records.all(): if 'description' in r.metadata: descriptions.update(r.metadata['description']) return '\n'.join(x for x in descriptions if x) def prepare_language(self, obj: Resource) -> str: languages = { n.language.name for n in obj.nominations.all() if n.language } for r in obj.imported_records.all(): if 'language' in r.metadata: languages.update(r.metadata['language']) return '\n'.join(x for x in languages if x) def prepare_tags(self, obj: Resource) -> List[str]: tags = set() for nom in obj.nominations.all(): tags.update([t.name for t in nom.tags.all()]) for rec in obj.imported_records.all(): try: tags.update(rec.metadata['tags']) except KeyError: pass return list(tags) # return ','.join(x for x in tags if x) def prepare_subject_headings(self, obj: Resource) -> List[str]: subject_headings: set = set() for nom in obj.nominations.all(): subject_headings.update(t.name for t in nom.subject_headings.all()) for rec in obj.imported_records.all(): try: subject_headings.update(rec.metadata['subject_headings']) except KeyError: pass return list(subject_headings)
class I4pProjectIndex(indexes.SearchIndex, indexes.Indexable): #text = indexes.MultiValueField(document=True, use_template=False) text = indexes.CharField(document=True, use_template=False) #title = indexes.CharField(model_attr='title') #baseline = indexes.CharField(model_attr='baseline') #For some reason MultiValueField doesn't work properly with whoosh #language_codes = indexes.CharField(indexed=True, stored=True) language_codes = indexes.MultiValueField(indexed=True, stored=True) #slug = indexes.CharField(model_attr='slug') #content_auto = indexes.EdgeNgramField(model_attr='title') best_of = indexes.BooleanField(model_attr='best_of') status = indexes.CharField(model_attr='status', null=True) sites = indexes.MultiValueField() tags = indexes.MultiValueField(indexed=True, stored=True) countries = indexes.MultiValueField(indexed=True, stored=True) has_team = indexes.BooleanField() has_needs = indexes.BooleanField() created = indexes.DateTimeField(model_attr='created') def get_model(self): return I4pProject def read_queryset(self, using=None): return I4pProject.objects.untranslated().all() def prepare_sites(self, obj): return [obj.id for obj in obj.site.all()] def prepare_language_codes(self, obj): return obj.get_available_languages() def prepare_text(self, obj): """ The textual content of the project """ languages = obj.get_available_languages() retval = [] for language in languages: translated_project = self.get_model().objects.language( language).get(pk=obj.pk) retval.append(translated_project.title) retval.append(translated_project.baseline) # Fetch questions questions_content = [] for topic in Topic.objects.language(language).filter( site_topics__projects=translated_project): questions_content.append(topic.label) for question in topic.questions.language( language).all().order_by('weight'): answers = Answer.objects.language(language).filter( project=translated_project, question=question) if len(answers) >= 1: #We only want the question text if there are answers questions_content.append(question.content) questions_content.extend( [answer.content for answer in answers]) retval.extend(questions_content) return '\n'.join(retval) def prepare_has_team(self, obj): """ If there is at least one user associated with this project """ return obj.members.count() > 0 def prepare_has_needs(self, obj): """ If the project has expressed needs """ #FIXME: django-hvad does NOT support foreign keys on translated models #At least we don't try to use it until this is ported properly translations = I4pProjectTranslation.objects.filter(master=obj.id) projectsupport_count = 0 for translated_project in translations: projectsupport_count += SpecificQuestion.objects.filter( type__in=SpecificQuestionType.objects.filter(type="pj-help"), object_id=translated_project.pk).count() projectsupport_count += SpecificQuestion.objects.filter( type__in=SpecificQuestionType.objects.filter(type="pj-need"), object_id=translated_project.pk).count() return projectsupport_count > 0 def prepare_tags(self, obj): """ Split tags by comma, strip and remove empty """ languages = obj.get_available_languages() taglist = [] for language in languages: translated_project = self.get_model().objects.language( language).get(pk=obj.pk) taglist.extend(translated_project.themes.split(',')) return set([tag.strip() for tag in taglist if tag.strip()]) def prepare_countries(self, obj): if obj.locations: return [location.country.code for location in obj.locations.all()] else: return None
class TendenciBaseSearchIndex(indexes.SearchIndex): text = indexes.CharField(document=True, use_template=True) # TendenciBaseModel Fields allow_anonymous_view = indexes.BooleanField(model_attr='allow_anonymous_view', null=True) allow_user_view = indexes.BooleanField(model_attr='allow_user_view', null=True) allow_member_view = indexes.BooleanField(model_attr='allow_member_view', null=True) # allow_anonymous_edit = indexes.BooleanField(model_attr='allow_anonymous_edit') allow_user_edit = indexes.BooleanField(model_attr='allow_user_edit', null=True) allow_member_edit = indexes.BooleanField(model_attr='allow_member_edit', null=True) creator = indexes.CharField(model_attr='creator', null=True) creator_username = indexes.CharField(model_attr='creator_username', null=True) owner = indexes.CharField(model_attr='owner', null=True) owner_username = indexes.CharField(model_attr='owner_username', null=True) status = indexes.BooleanField(model_attr='status', null=True) status_detail = indexes.CharField(model_attr='status_detail') create_dt = indexes.DateTimeField(model_attr='create_dt', null=True) update_dt = indexes.DateTimeField(model_attr='update_dt', null=True) # permission fields primary_key = indexes.IntegerField(model_attr='pk') users_can_view = indexes.MultiValueField(null=True) groups_can_view = indexes.MultiValueField(null=True) # add order field for sorting. the subclasses can override # the prepare_order method to sort by a different field order = indexes.DateTimeField() def get_model(self): return None def prepare_allow_anonymous_view(self, obj): if is_whoosh(): try: temp = int(obj.allow_anonymous_view) except TypeError: temp = 0 return temp return obj.allow_anonymous_view def prepare_allow_user_view(self, obj): if is_whoosh(): try: temp = int(obj.allow_user_view) except TypeError: temp = 0 return temp return obj.allow_user_view def prepare_allow_member_view(self, obj): if is_whoosh(): try: temp = int(obj.allow_member_view) except TypeError: temp = 0 return temp return obj.allow_member_view def prepare_allow_user_edit(self, obj): if is_whoosh(): try: temp = int(obj.allow_user_edit) except TypeError: temp = 0 return temp return obj.allow_user_edit def prepare_allow_member_edit(self, obj): if is_whoosh(): try: temp = int(obj.allow_member_edit) except TypeError: temp = 0 return temp return obj.allow_member_edit def prepare_status(self, obj): if is_whoosh(): try: temp = int(obj.status) except TypeError: temp = 0 return temp return obj.status def prepare_order(self, obj): return obj.create_dt def get_updated_field(self): return 'update_dt' def prepare_users_can_view(self, obj): """ This needs to be overwritten if 'view' permission label does not follow the standard convention: (app_label).view_(module_name) """ return ObjectPermission.objects.users_with_perms('%s.view_%s' % (obj._meta.app_label, obj._meta.model_name), obj) def prepare_groups_can_view(self, obj): """ This needs to be overwritten if 'view' permission label does not follow the standard convention: (app_label).view_(module_name) """ return ObjectPermission.objects.groups_with_perms('%s.view_%s' % (obj._meta.app_label, obj._meta.model_name), obj)
class UserIndex(indexes.RealTimeSearchIndex): text = indexes.CharField(document=True, model_attr='get_full_name')
class CourseRunIndex(BaseCourseIndex, indexes.Indexable): model = CourseRun course_key = indexes.CharField(model_attr='course__key', stored=True) org = indexes.CharField() number = indexes.CharField() status = indexes.CharField(model_attr='status', faceted=True) start = indexes.DateTimeField(model_attr='start', null=True, faceted=True) end = indexes.DateTimeField(model_attr='end', null=True) enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True) enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True) announcement = indexes.DateTimeField(model_attr='announcement', null=True) min_effort = indexes.IntegerField(model_attr='min_effort', null=True) max_effort = indexes.IntegerField(model_attr='max_effort', null=True) weeks_to_complete = indexes.IntegerField(model_attr='weeks_to_complete', null=True) language = indexes.CharField(null=True, faceted=True) transcript_languages = indexes.MultiValueField(faceted=True) pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True) marketing_url = indexes.CharField(null=True) slug = indexes.CharField(model_attr='slug', null=True) seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True) type = indexes.CharField(model_attr='type', null=True, faceted=True) image_url = indexes.CharField(model_attr='card_image_url', null=True) partner = indexes.CharField(null=True, faceted=True) program_types = indexes.MultiValueField() published = indexes.BooleanField(null=False, faceted=True) hidden = indexes.BooleanField(model_attr='hidden', faceted=True) mobile_available = indexes.BooleanField(model_attr='mobile_available', faceted=True) authoring_organization_uuids = indexes.MultiValueField() staff_uuids = indexes.MultiValueField() subject_uuids = indexes.MultiValueField() has_enrollable_paid_seats = indexes.BooleanField(null=False) paid_seat_enrollment_end = indexes.DateTimeField(null=True) def prepare_aggregation_key(self, obj): # Aggregate CourseRuns by Course key since that is how we plan to dedup CourseRuns on the marketing site. return 'courserun:{}'.format(obj.course.key) def prepare_has_enrollable_paid_seats(self, obj): return obj.has_enrollable_paid_seats() def prepare_paid_seat_enrollment_end(self, obj): return obj.get_paid_seat_enrollment_end() def prepare_partner(self, obj): return obj.course.partner.short_code def prepare_published(self, obj): return obj.status == CourseRunStatus.Published def prepare_language(self, obj): return self._prepare_language(obj.language) def prepare_number(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.course def prepare_org(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.org def prepare_transcript_languages(self, obj): return [self._prepare_language(language) for language in obj.transcript_languages.all()] def prepare_marketing_url(self, obj): return obj.marketing_url def prepare_program_types(self, obj): return obj.program_types def prepare_staff_uuids(self, obj): return [str(staff.uuid) for staff in obj.staff.all()] def prepare_subject_uuids(self, obj): return [str(subject.uuid) for subject in obj.subjects.all()]
class EventIndex(TendenciBaseSearchIndex, indexes.Indexable): title = indexes.CharField(model_attr='title') description = indexes.CharField(model_attr='description') start_dt = indexes.DateTimeField(model_attr='start_dt') end_dt = indexes.DateTimeField(model_attr='end_dt') on_weekend = indexes.BooleanField(model_attr='on_weekend', null=True) # fields for sorting events that span multiple days hour = indexes.IntegerField() minute = indexes.IntegerField() # event type id type_id = indexes.IntegerField(null=True) # amount of registrations spots left spots_taken = indexes.IntegerField() # number of days the event is active number_of_days = indexes.IntegerField() # RSS fields can_syndicate = indexes.BooleanField(null=True) order = indexes.DateTimeField() @classmethod def get_model(self): return Event def prepare_description(self, obj): description = obj.description description = strip_tags(description) description = strip_entities(description) return description def prepare_hour(self, obj): return int(obj.start_dt.hour) def prepare_minute(self, obj): return int(obj.start_dt.minute) def prepare_type_id(self, obj): if obj.type: return obj.type.pk else: return None def prepare_spots_taken(self, obj): if hasattr(obj, 'spots_taken'): return obj.spots_taken else: return count_event_spots_taken(obj) def prepare_number_of_days(self, obj): return obj.number_of_days() def prepare_can_syndicate(self, obj): return obj.allow_anonymous_view and obj.status == 1 \ and obj.status_detail == 'active' def prepare_order(self, obj): return obj.start_dt
class ReviewRequestIndex(BaseSearchIndex, indexes.Indexable): """A Haystack search index for Review Requests.""" model = ReviewRequest local_site_attr = 'local_site_id' # We shouldn't use 'id' as a field name because it's by default reserved # for Haystack. Hiding it will cause duplicates when updating the index. review_request_id = indexes.IntegerField(model_attr='display_id') summary = indexes.CharField(model_attr='summary') description = indexes.CharField(model_attr='description') testing_done = indexes.CharField(model_attr='testing_done') commit_id = indexes.EdgeNgramField(model_attr='commit', null=True) bug = indexes.CharField(model_attr='bugs_closed') username = indexes.CharField(model_attr='submitter__username') author = indexes.CharField() last_updated = indexes.DateTimeField(model_attr='last_updated') url = indexes.CharField(model_attr='get_absolute_url') file = indexes.CharField() # These fields all contain information needed to perform queries about # whether a review request is accessible by a given user. private = indexes.BooleanField() private_repository_id = indexes.IntegerField() private_target_groups = indexes.MultiValueField() target_users = indexes.MultiValueField() def get_model(self): """Returns the Django model for this index.""" return ReviewRequest def get_updated_field(self): return 'last_updated' def index_queryset(self, using=None): """Index only public pending and submitted review requests.""" return (self.get_model().objects.public( status=None, show_all_local_sites=True, show_inactive=True, filter_private=False).select_related( 'diffset_history', 'local_site', 'repository', 'submitter', 'submitter__profile').prefetch_related( 'diffset_history__diffsets__files', 'target_groups', 'target_people')) def prepare_file(self, obj): return set([(filediff.source_file, filediff.dest_file) for diffset in obj.diffset_history.diffsets.all() for filediff in diffset.files.all()]) def prepare_private(self, review_request): """Prepare the private flag for the index. This will be set to true if the review request isn't generally accessible to users. """ return not review_request.is_accessible_by(AnonymousUser(), silent=True) def prepare_private_repository_id(self, review_request): """Prepare the private repository ID, if any, for the index. If there's no repository, or it's public, 0 will be returned instead. """ if review_request.repository and not review_request.repository.public: return review_request.repository_id else: return 0 def prepare_private_target_groups(self, review_request): """Prepare the list of invite-only target groups for the index. If there aren't any invite-only groups associated, ``[0]`` will be returned. This allows queries to be performed that check that none of the groups are private, since we can't query against empty lists. """ return [ group.pk for group in review_request.target_groups.all() if group.invite_only ] or [0] def prepare_target_users(self, review_request): """Prepare the list of target users for the index. If there aren't any target users, ``[0]`` will be returned. This allows queries to be performed that check that there aren't any users in the list, since we can't query against empty lists. """ return [user.pk for user in review_request.target_people.all()] or [0] def prepare_author(self, review_request): """Prepare the author field. Args: review_request (reviewboard.reviews.models.review_request. ReviewRequest): The review request being indexed. Returns: unicode: Either the author's full name (if their profile is public) or an empty string. """ user = review_request.submitter profile = user.get_profile(cached_only=True) if profile is None or profile.is_private: return '' return user.get_full_name()
class BaseSearchIndex(indexes.SearchIndex): """Base class for a search index. This sets up a few common fields we want all indexes to include. """ #: The model to index. model = None #: The local site attribute on the model. #: #: For ForeignKeys, this should be the name of the ID field, as in #: 'local_site_id'. For ManyToManyFields, it should be the standar field #: name. local_site_attr = None # Common fields for all search indexes. text = indexes.CharField(document=True, use_template=True) local_sites = indexes.MultiValueField(null=True) NO_LOCAL_SITE_ID = 0 def get_model(self): """Return the model for this index.""" return self.model def prepare_local_sites(self, obj): """Prepare the list of local sites for the search index. This will take any associated local sites on the object and store them in the index as a list. The search view can then easily look up values in the list, regardless of the type of object. If the object is not a part of a local site, the list will be ``[0]``, indicating no local site. """ if not self.local_site_attr: raise ImproperlyConfigured('local_site_attr must be set on %r' % self.__class__) if not hasattr(obj, self.local_site_attr): raise ImproperlyConfigured( '"%s" is not a valid local site attribute on %r' % (self.local_site_attr, obj.__class__)) local_sites = getattr(obj, self.local_site_attr, None) if self.local_site_attr.endswith('_id'): # This is from a ForeignKey. We're working with a numeric ID. if local_sites is not None: results = [local_sites] else: results = [self.NO_LOCAL_SITE_ID] else: # This is most likely a ManyToManyField. Anything else is an # error. # # We want to loop through the actual entries and not the primary # keys. The caller is responsible for doing a prefetch_related(). results = [local_site.pk for local_site in local_sites.all() ] or [self.NO_LOCAL_SITE_ID] # Convert these all to strings. This is what MultiValueField would # normally do if we didn't prepare it, and is needed for the kinds of # comparisons we perform when using Elasticsearch 7.x+. return [str(_pk) for _pk in results]
class PortIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) name = indexes.NgramField(model_attr='name', boost=2.0) name_lower = indexes.CharField(indexed=False, stored=True) name_length = indexes.IntegerField(indexed=False) maintainers = indexes.MultiValueField(boost=1.2, faceted=True) description = indexes.CharField(model_attr='description', boost=1.5) variants = indexes.MultiValueField(faceted=True) livecheck_broken = indexes.BooleanField() livecheck_outdated = indexes.BooleanField() nomaintainer = indexes.BooleanField() active = indexes.BooleanField(model_attr='active') categories = indexes.MultiValueField(faceted=True) version = indexes.CharField(model_attr='version', indexed=False) files = indexes.MultiValueField() def get_model(self): return Port def get_updated_field(self): return 'updated_at' def prepare_name_length(self, obj): return len(obj.name) def prepare_name_lower(self, obj): return str(obj.name).lower() def prepare_maintainers(self, obj): return [m.github for m in obj.maintainers.all()] def prepare_nomaintainer(self, obj): return False if obj.maintainers.all().count() > 0 else True def prepare_variants(self, obj): return [v.variant for v in obj.variants.all()] def prepare_livecheck_broken(self, obj): if hasattr(obj, 'livecheck'): return False if obj.livecheck.error is None else True else: return False def prepare_livecheck_outdated(self, obj): if hasattr(obj, 'livecheck'): return obj.livecheck.has_updates else: return False def prepare_categories(self, obj): return [c.name for c in obj.categories.all()] def prepare_files(self, obj): port_name = obj.name latest_build = BuildHistory.objects.filter( port_name__iexact=port_name, status="build successful").order_by( '-time_start').prefetch_related('files').first() if latest_build: return [f.file for f in latest_build.files.all()] else: return []
class MediaCollectionIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) item_acces = indexes.CharField(model_attr='public_access', faceted=True) item_status = indexes.CharField(model_attr='document_status', faceted=True) digitized = indexes.BooleanField(default=False, faceted=True) media_type = indexes.CharField(model_attr='media_type', null='None', faceted=True) recording_context = indexes.CharField(model_attr='recording_context', default='', faceted=True) physical_format = indexes.CharField(model_attr='physical_format', default='', faceted=True) #content_auto = indexes.EdgeNgramField(model_attr='content') #advance search title = indexes.CharField(model_attr='title') code = KeywordField(model_attr='code', default='') location_principal = indexes.CharField(default='', boost=1.05) location_relation = indexes.CharField() ethnic_group = indexes.CharField(default='') instruments = indexes.CharField(default='') collectors = indexes.CharField(model_attr='collector', default='') recorded_from_date = indexes.DateField(model_attr='recorded_from_year', null=True) recorded_to_date = indexes.DateField(model_attr='recorded_to_year', null=True) year_published = indexes.IntegerField(model_attr='year_published', default='') def prepare_digitized(self, obj): return obj.has_mediafile def get_model(self): return MediaCollection def prepare_location_principal(self, obj): collec_location = [] for item in obj.items.all(): location = [] if item.location is not None: collec_location.append(item.location.name) return u"".join('|' + location for location in collec_location) def prepare_location_relation(self, obj): collec_location = [] for item in obj.items.all(): location = [] if item.location is not None: location_alias = LocationAlias.objects.filter( location__name=item.location) location_rela = LocationRelation.objects.filter( location__name=item.location) for rela in location_rela: location.append(rela.ancestor_location.name) for alias in location_alias: location.append(alias.alias) if item.location.current_location is not None: location.append(item.location.current_location.name) for name in location: if name and not name in collec_location: collec_location.append(name) return u"".join('|' + location for location in collec_location) def prepare_ethnic_group(self, obj): return "%s" % obj.ethnic_groups() def prepare_instruments(self, obj): instruments = [] items = obj.items.all() for item in items: materials = MediaItemPerformance.objects.all().filter( media_item__exact=item) for material in materials: if material.instrument and not material.instrument in instruments: instruments.append(material.instrument.name) if material.alias and not material.alias in instruments: instruments.append(material.alias.name) return u"".join('|' + instru for instru in instruments) def prepare_recorded_from_date(self, obj): if obj.recorded_from_year != 0: return datetime.date(int(obj.recorded_from_year), 01, 01) else: return None def prepare_recorded_to_date(self, obj): if obj.recorded_to_year != 0: return datetime.date(int(obj.recorded_to_year), 01, 01) else: return None
class ProfileIndex(indexes.SearchIndex): description = indexes.CharField(document=True, use_template=True, model_attr='user') full_name = indexes.CharField(model_attr='name')
class MediaItemIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) item_acces = indexes.CharField(model_attr='collection__public_access', faceted=True) item_status = indexes.CharField(model_attr='collection__document_status', faceted=True) digitized = indexes.BooleanField(default=False, faceted=True) media_type = indexes.CharField(model_attr='media_type', null='None', faceted=True) recording_context = indexes.CharField( model_attr='collection__recording_context', default='', faceted=True) physical_format = indexes.CharField( model_attr='collection__physical_format', default='', faceted=True) #content_auto = indexes.EdgeNgramField(model_attr='content') #advance search title = indexes.CharField(model_attr='title') code = KeywordField(model_attr='code', default='') location_principal = indexes.CharField(null='None', boost=1.05) location_relation = indexes.CharField() ethnic_group = indexes.CharField(model_attr='ethnic_group', default='') instruments = indexes.CharField(default='') collectors = indexes.CharField(model_attr='collector', default='') recorded_from_date = indexes.DateField(model_attr='recorded_from_date', null='None') recorded_to_date = indexes.DateField(model_attr='recorded_to_date', null='None') year_published = indexes.IntegerField( model_attr='collection__year_published', default='') def prepare_digitized(self, obj): if obj.file.name: return True elif '/' in obj.url: return True else: return False def get_model(self): return MediaItem def prepare_location_principal(self, obj): if obj.location is not None: return u"".join(obj.location.name) else: return None def prepare_location_relation(self, obj): location = [] if obj.location is not None: location_alias = LocationAlias.objects.filter( location__name=obj.location) location_rela = LocationRelation.objects.filter( location__name=obj.location) for rela in location_rela: location.append(rela.ancestor_location.name) for alias in location_alias: location.append(alias.alias) if obj.location.current_location is not None: location.append(obj.location.current_location.name) #print u"".join(' ' + local for local in location).encode("utf-8") #print u"%s".encode("utf-8") % location #print [local for local in location] return u"".join('|' + local for local in location) def prepare_instruments(self, obj): item = MediaItemPerformance.objects.all().filter(media_item__exact=obj) instruments = [] for material in item: if material.instrument is not None: instruments.append(material.instrument.name) if material.alias is not None: instruments.append(material.alias.name) return u"".join('|' + instru for instru in instruments) def prepare_collectors(self, obj): collectors = [] collectors.append(obj.collection.collector) collectors.append(obj.collector) return u"".join('; ' + collector for collector in collectors)
class SolrMockTagSearchIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, model_attr="name") def get_model(self): return MockTag
class ProductIndex(indexes.SearchIndex, indexes.Indexable): # Search text text = indexes.EdgeNgramField( document=True, use_template=True, template_name='oscar/search/indexes/product/item_text.txt') upc = indexes.CharField(model_attr="upc", null=True) title = indexes.EdgeNgramField(model_attr='title', null=True) title_exact = indexes.CharField(model_attr='title', null=True, indexed=False) # Fields for faceting product_class = indexes.CharField(null=True, faceted=True) category = indexes.MultiValueField(null=True, faceted=True) price = indexes.DecimalField(null=True, faceted=True) num_in_stock = indexes.IntegerField(null=True, faceted=True) rating = indexes.IntegerField(null=True, faceted=True) # Spelling suggestions suggestions = indexes.FacetCharField() date_created = indexes.DateTimeField(model_attr='date_created') date_updated = indexes.DateTimeField(model_attr='date_updated') def get_model(self): return get_model('catalogue', 'Product') def index_queryset(self, using=None): # Only index browsable products (not each individual child product) return self.get_model().browsable.order_by('-date_updated') def read_queryset(self, using=None): return self.get_model().browsable.base_queryset() def prepare_product_class(self, obj): return obj.get_product_class().name def prepare_category(self, obj): categories = obj.categories.all() if len(categories) > 0: return [category.full_name for category in categories] def prepare_rating(self, obj): if obj.rating is not None: return int(obj.rating) # Pricing and stock is tricky as it can vary per customer. However, the # most common case is for customers to see the same prices and stock levels # and so we implement that case here. def prepare_price(self, obj): result = None if obj.is_parent: result = strategy.fetch_for_parent(obj) elif obj.has_stockrecords: result = strategy.fetch_for_product(obj) if result: if result.price.is_tax_known: return result.price.incl_tax return result.price.excl_tax def prepare_num_in_stock(self, obj): if obj.is_parent: # Don't return a stock level for parent products return None elif obj.has_stockrecords: result = strategy.fetch_for_product(obj) return result.stockrecord.net_stock_level def prepare(self, obj): prepared_data = super(ProductIndex, self).prepare(obj) # We use Haystack's dynamic fields to ensure that the title field used # for sorting is of type "string'. if is_solr_supported(): prepared_data['title_s'] = prepared_data['title'] # Use title to for spelling suggestions prepared_data['suggestions'] = prepared_data['text'] return prepared_data def get_updated_field(self): """ Used to specify the field used to determine if an object has been updated Can be used to filter the query set when updating the index """ return 'date_updated'
class StatementIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, model_attr='text_plain') searchtext = indexes.CharField(stored=False, use_template=True) date = indexes.DateTimeField(model_attr='time') politician = indexes.CharField(use_template=True) politician_id = indexes.CharField( model_attr='member__politician__identifier', null=True) who_hocid = indexes.IntegerField(model_attr='who_hocid', null=True) party = indexes.CharField(model_attr='member__party__short_name', null=True) province = indexes.CharField(model_attr='member__riding__province', null=True) topic = indexes.CharField(model_attr='topic') url = indexes.CharField(model_attr='get_absolute_url', indexed=False) doc_url = indexes.CharField(model_attr='document__get_absolute_url') committee = indexes.CharField(model_attr='committee_name') committee_slug = indexes.CharField(model_attr='committee_slug') doctype = indexes.CharField(null=True) def get_model(self): return Statement def index_queryset(self): return Statement.objects.all().prefetch_related( 'member__politician', 'member__party', 'member__riding', 'document', 'document__committeemeeting__committee').order_by('-date') def prepare_doctype(self, obj): if obj.committee_name: return 'committee' else: return 'debate'
class MXIndex(CydnsIndex, indexes.Indexable): server = indexes.CharField(model_attr='server') def get_model(self): return MX
class WhooshAutocompleteMockModelSearchIndex(indexes.SearchIndex): text = indexes.CharField(model_attr='foo', document=True) name = indexes.CharField(model_attr='author') pub_date = indexes.DateField(model_attr='pub_date') text_auto = indexes.EdgeNgramField(model_attr='foo') name_auto = indexes.EdgeNgramField(model_attr='author')
class ProjectIndex(CelerySearchIndex, indexes.Indexable): def get_model(self): return Project def get_updated_field(self): return 'updated' # needed text = indexes.CharField(document=True, use_template=True) acronym = indexes.CharField(indexed=True, stored=True, faceted=True) acronym_l = indexes.CharField(indexed=False, stored=True) cner_notes = indexes.CharField(indexed=True, stored=True) contacts = indexes.MultiValueField(indexed=True, stored=True, faceted=True) company_personnel = indexes.MultiValueField(indexed=True, stored=True, faceted=True) description = indexes.CharField(indexed=True, stored=True, faceted=True) disease_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) end_year = indexes.IntegerField(indexed=True, stored=True, faceted=True) erp_notes = indexes.CharField(indexed=True, stored=True, faceted=True) funding_sources = indexes.MultiValueField(indexed=True, stored=True, faceted=True) gene_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) has_cner = indexes.BooleanField(indexed=True, stored=True, faceted=True) has_erp = indexes.BooleanField(indexed=True, stored=True, faceted=True) has_legal_documents = indexes.BooleanField(indexed=True, stored=True, faceted=True) umbrella_project = indexes.CharField(indexed=True, stored=True, faceted=True) phenotype_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) project_web_page = indexes.CharField(indexed=True, stored=True, faceted=True) publications = indexes.MultiValueField(indexed=True, stored=True) start_date = indexes.DateField(indexed=True, stored=True) start_year = indexes.IntegerField(indexed=True, stored=True, faceted=True) study_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) title = indexes.CharField(indexed=True, stored=True, faceted=True) title_l = indexes.CharField(indexed=False, stored=True) local_custodians = indexes.MultiValueField(indexed=True, stored=True, faceted=True) def prepare_acronym_l(self, obj): if obj.acronym: return obj.acronym.lower().strip() def prepare_title_l(self, obj): if obj.title: return obj.title.lower().strip() def prepare_contacts(self, obj): return [str(o) for o in obj.contacts.all()] def prepare_company_personnel(self, obj): return [str(o) for o in obj.company_personnel.all()] def prepare_funding_sources(self, obj): return [str(o) for o in obj.funding_sources.all()] def prepare_publications(self, obj): return [str(o) for o in obj.publications.all()] def prepare_local_custodians(self, obj): return [str(o) for o in obj.local_custodians.all()] def prepare_has_legal_documents(self, obj): return obj.legal_documents.exists() def prepare_acronym(self, obj): return obj.acronym def prepare_cner_notes(self, obj): return obj.cner_notes def prepare_description(self, obj): return obj.description def prepare_disease_terms(self, obj): return [str(o) for o in obj.disease_terms.all()] def prepare_end_year(self, obj): if obj.end_date: return obj.end_date.year else: return None def prepare_erp_notes(self, obj): return obj.erp_notes def prepare_gene_terms(self, obj): return [str(o) for o in obj.gene_terms.all()] def prepare_has_cner(self, obj): return obj.has_cner def prepare_has_erp(self, obj): return obj.has_erp def prepare_umbrella_project(self, obj): return obj.umbrella_project def prepare_phenotype_terms(self, obj): return [str(o) for o in obj.phenotype_terms.all()] def prepare_project_web_page(self, obj): return obj.project_web_page def prepare_start_year(self, obj): if obj.start_date: return obj.start_date.year else: return None def prepare_start_date(self, obj): if obj.start_date: return obj.start_date else: return None def prepare_study_terms(self, obj): return [str(o) for o in obj.study_terms.all()] def prepare_title(self, obj): return obj.title