class DatasetIndex(indexes.ModelSearchIndex, indexes.Indexable): title = indexes.CharField(model_attr='title', boost=1.25) description = indexes.CharField(model_attr='description', boost=1.125) group_name = indexes.CharField(model_attr='group_name', faceted=True) access_type = indexes.CharField(model_attr='access_type', faceted=True) states = indexes.FacetMultiValueField(model_attr='states') division_names = indexes.FacetMultiValueField(model_attr='division_names') sectors = indexes.FacetMultiValueField(model_attr='sectors') formats = indexes.MultiValueField(model_attr='formats', boost=0.6) tags = indexes.MultiValueField(model_attr='tags', boost=0.8) class Meta: model = Dataset excludes = ['uuid', 'created_at', 'updated_at', 'formats', 'tags'] def prepare_states(self, object): states_list = list(object.states_expanded()) return states_list def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return self.get_model().objects.filter(updated_at__lte=timezone.now()) def get_updated_field(self): return 'updated_at'
class PublicBodyIndex(SearchIndex, indexes.Indexable): text = indexes.EdgeNgramField(document=True, use_template=True) name = indexes.CharField(model_attr='name', boost=1.5) name_auto = SuggestField(model_attr='all_names', boost=2.5) jurisdiction = indexes.FacetCharField(model_attr='jurisdiction__name', default='') classification = indexes.FacetMultiValueField() categories = indexes.FacetMultiValueField() def get_model(self): return PublicBody def index_queryset(self, **kwargs): """Used when the entire index for model is updated.""" return self.get_model().objects.get_for_search_index() def prepare_classification(self, obj): if obj.classification is None: return [] return [obj.classification.name ] + [c.name for c in obj.classification.get_ancestors()] def prepare_categories(self, obj): cats = obj.categories.all() return [o.name for o in cats ] + [c.name for o in cats for c in o.get_ancestors()]
class InitiativeIndex(AbstractIndex, indexes.ModelSearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) # facets years = indexes.FacetMultiValueField() recipients = indexes.FacetMultiValueField() agency = indexes.FacetCharField() aid_types = indexes.FacetMultiValueField() channels = indexes.FacetMultiValueField() finance_type = indexes.FacetCharField() sectors = indexes.FacetMultiValueField() def prepare_years(self, obj): return obj.years_range() def prepare_recipients(self, obj): return self._prepare_codelist(obj.recipients(), roots=False) def prepare_agency(self, obj): agency = obj.agency() return agency.code if agency else None def prepare_aid_types(self, obj): return self._prepare_codelist(obj.aid_types()) def prepare_channels(self, obj): return self._prepare_codelist(obj.channels()) def prepare_finance_type(self, obj): finance_type = obj.finance_type() return finance_type.code if finance_type else None def prepare_sectors(self, obj): return self._prepare_codelist(obj.sectors()) class Meta: model = Initiative # fields = ['years', 'recipients', 'agency', 'aid_types', # 'channels', 'finance_type', 'sectors', # 'code', 'start_year', 'end_year', 'has_focus'] excludes = [ 'code', 'title', 'total_project_costs', 'loan_amount_approved', 'grant_amount_approved', 'photo_set', 'document_set', 'updated_at', 'created_at' ] for f in Initiative._meta.fields: if f.attname.endswith('_temp') or f.attname[-3:] in ('_it', '_en'): excludes.append(f.attname)
class EventIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) categories = indexes.FacetMultiValueField() title = indexes.CharField(model_attr='title') end_date = indexes.DateField( model_attr='end_date', default=lambda: datetime.now() + timedelta(days=(30 * 365))) # If no expiry, expire after 30 years city = indexes.CharField(model_attr='venue__city', faceted=True, null=True) neighborhood = indexes.CharField(model_attr='venue__neighborhood', faceted=True, null=True) point = indexes.LocationField(model_attr='venue__point', null=True) # We add this for autocomplete. title_auto = indexes.EdgeNgramField(use_template=True) def get_model(self): return Event def prepare_categories(self, obj): return [category.name for category in obj.category.all()] def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return self.get_model().objects.filter(publish=True)
class PublicationIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.NgramField(document=True, use_template=True) title = indexes.NgramField(model_attr='titre') bureaux = indexes.FacetMultiValueField(null=True, stored=True) annee = indexes.FacetField(stored=True, null=True) section = indexes.FacetField(stored=True, null=True) date_pub = indexes.DateField(model_attr='date_pub', null=True) def prepare_bureaux(self, obj): try: return [b.nom for b in obj.bureau.all()] except ObjectDoesNotExist as e: print(e) return [u'Non précisé'] def prepare_annee(self, obj): if obj.date_pub is not None: return str(obj.date_pub.year) def get_model(self): return Publication def prepare_section(self, obj): return u"Publication" def index_queryset(self, using=None): return Publication.objects.filter( status__in=[3, 5, 6]).order_by("date_pub")
class PublicBodyIndex(SearchIndex, indexes.Indexable): text = indexes.EdgeNgramField(document=True, use_template=True) name = indexes.CharField(model_attr='name', boost=1.5) name_auto = indexes.NgramField(model_attr='name') jurisdiction = indexes.FacetCharField(model_attr='jurisdiction__name', default='') tags = indexes.FacetMultiValueField() url = indexes.CharField(model_attr='get_absolute_url') def get_model(self): return PublicBody def index_queryset(self, **kwargs): """Used when the entire index for model is updated.""" return self.get_model().objects.get_for_search_index() def prepare_tags(self, obj): return [t.name for t in obj.tags.all()] def prepare(self, obj): data = super(PublicBodyIndex, self).prepare(obj) if obj.classification in PUBLIC_BODY_BOOSTS: data['boost'] = PUBLIC_BODY_BOOSTS[obj.classification] print("Boosting %s at %f" % (obj, data['boost'])) return data
class CandidateIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) first_name = indexes.CharField(indexed=True) party = indexes.FacetCharField(indexed=True) college = indexes.FacetCharField(indexed=True) state = indexes.FacetCharField(indexed=True) issues = indexes.FacetMultiValueField(indexed=True) race = indexes.FacetCharField(indexed=True) race_type = indexes.FacetCharField(indexed=True) women = indexes.BooleanField(indexed=True) active = indexes.BooleanField(indexed=True) random = indexes.CharField() def get_model(self): return Candidate def prepare_first_name(self, obj): return obj.first_name def prepare_issues(self, obj): return [(issue.issue.parent_name) for issue in obj.issues.all()] or None def prepare_party(self, obj): return obj.party def prepare_state(self, obj): return obj.state def prepare_race(self, obj): if obj.race: return obj.race.title def prepare_active(self, obj): return obj.active def prepare_women(self, obj): if obj.man: return False return True def prepare_race_type(self, obj): if obj.race: return obj.race.race_type def prepare_random(self, obj): return random.randint(0, 99999999) def prepare_college(self, obj): if obj.college: return obj.college.name return None def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return self.get_model().objects.all()
class OrganizationIndex(CelerySearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) created = indexes.DateTimeField(model_attr='created') modified = indexes.DateTimeField(model_attr='modified') state = indexes.FacetCharField(model_attr='state') tags = indexes.FacetMultiValueField() def prepare_tags(self, my_model): return [tag.id for tag in my_model.tags.all()] def get_model(self): return models.Organization
class AufIndex(AldrynIndexBase): index_title = True text = indexes.NgramField(document=True, use_template=False) title = indexes.NgramField(stored=True, indexed=False) description = indexes.NgramField(indexed=False, stored=True) bureaux = indexes.FacetMultiValueField(stored=True, null=True) annee = indexes.FacetField(stored=True, null=True) def prepare_bureaux(self, obj): try: return [b.nom for b in obj.page.bureauextension.bureau.all()] except ObjectDoesNotExist, e: for path in BUREAU_SLUGS.keys(): if path in obj.page.get_absolute_url(): return [BUREAU_SLUGS[path]] return [u'Non précisé']
class ProjectIndex(AbstractIndex, indexes.ModelSearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) # facets years = indexes.FacetMultiValueField() # facets for code-lists recipient = indexes.FacetCharField() agencies = indexes.FacetMultiValueField() aid_types = indexes.FacetMultiValueField() channels = indexes.FacetMultiValueField() finance_types = indexes.FacetMultiValueField() sectors = indexes.FacetMultiValueField() def prepare_recipient(self, obj): return obj.recipient.code def prepare_agencies(self, obj): return self._prepare_codelist(obj.agencies(), roots=False) def prepare_aid_types(self, obj): return self._prepare_codelist(obj.aid_types()) def prepare_channels(self, obj): return self._prepare_codelist(obj.channels()) def prepare_finance_types(self, obj): return self._prepare_codelist(obj.finance_types()) def prepare_sectors(self, obj): return self._prepare_codelist(obj.sectors()) def get_facets_counts(self): return self.objects.facet('year').facet_counts().get('fields', {}) class Meta: model = Project # fields = ['years', 'markers', 'recipient', 'agency', 'aid_type', # 'channel', 'finance_type', 'sector', 'initiative', # 'crsid', 'start_year', 'end_year', 'has_focus'] excludes = [ 'is_suspended', 'status', 'expected_start_year', 'expected_completion_year', 'expected_start_date', 'completion_date', 'last_update', 'outcome', 'beneficiaries', 'beneficiaries_female', 'total_project_costs', 'other_financiers', 'loan_amount_approved', 'grant_amount_approved', 'counterpart_authority', 'email', 'location', 'description', 'title', 'title_it', 'title_en', 'description_it', 'description_en', 'number', 'outcome_it', 'beneficiaries_it', 'other_financiers_it', 'counterpart_authority_it', 'location_it', 'outcome_en', 'beneficiaries_en', 'other_financiers_en', 'counterpart_authority_en', 'location_en' ]
class AufIndex(indexes.SearchIndex): text = indexes.NgramField(document=True, use_template=True) title = indexes.NgramField(model_attr='titre') bureaux = indexes.FacetMultiValueField(null=True, stored=True) annee = indexes.FacetField(stored=True, null=True) section = indexes.FacetField(stored=True, null=True) partenaire = indexes.FacetField(stored=True, null=True) date_pub = indexes.DateField(model_attr='date_pub', null=True) def prepare_bureaux(self, obj): regions = [] try: if obj.bureau.all( ).count == 0 or obj.status == "3" or obj.status == "5": regions.append(u'International') return regions + [b.nom for b in obj.bureau.all()] except ObjectDoesNotExist as e: print(e) return [] def prepare_annee(self, obj): if obj.date_pub is not None: return str(obj.date_pub.year)
class QuestionIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) content_auto = indexes.EdgeNgramField(model_attr='question_text') state = indexes.IntegerField(model_attr='state') region = indexes.MultiValueField() organisation = indexes.CharField(model_attr='organisation__id') #Facets state_facet = indexes.FacetIntegerField(model_attr = 'state') institution_facet = indexes.FacetMultiValueField() promotor_facet = indexes.FacetMultiValueField() faculty_facet = indexes.FacetMultiValueField() education_facet = indexes.FacetMultiValueField() subject_facet = indexes.FacetMultiValueField() key_word_facet = indexes.FacetMultiValueField() def prepare_region(self, obj): return [region.region for region in obj.region.all()] def prepare_institution_facet(self,obj): return [institution.name for institution in obj.institution.all()] def prepare_promotor_facet(self,obj): return ['{0} {1}'.format(promotor.first_name, promotor.last_name) for promotor in obj.promotor.all()] def prepare_faculty_facet(self,obj): return [faculty.name for faculty in obj.faculty.all()] def prepare_education_facet(self, obj): return [education.education for education in obj.education.all()] def prepare_subject_facet(self,obj): return [subject.subject for subject in obj.question_subject.all()] def prepare_key_word_facet(self,obj): return [keyword.key_word for keyword in obj.keyword.all()] def get_model(self): return Question
class StoryIndex(indexes.RealTimeSearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) author = indexes.FacetCharField(model_attr='author') published = indexes.FacetDateTimeField(model_attr='published') created = indexes.FacetDateTimeField(model_attr='created') last_edited = indexes.FacetDateTimeField(model_attr='last_edited') # TODO: Use a meta class to dynamically populate these from "official" # tag sets topic_ids = indexes.FacetMultiValueField() organization_ids = indexes.FacetMultiValueField() project_ids = indexes.FacetMultiValueField() language_ids = indexes.FacetMultiValueField() place_ids = indexes.FacetMultiValueField() points = GeoHashMultiValueField() num_points = indexes.IntegerField() def get_model(self): return Story def prepare_topic_ids(self, obj): return [topic.id for topic in obj.topics.all()] def prepare_organization_ids(self, obj): return [ organization.organization_id for organization in obj.organizations.all() ] def prepare_project_ids(self, obj): return [project.project_id for project in obj.projects.all()] def prepare_language_ids(self, obj): return obj.get_languages() def prepare_place_ids(self, obj): return [place.place_id for place in obj.inherited_places] def prepare_points(self, obj): return ["%s,%s" % (point[0], point[1]) for point in obj.points] def prepare_num_points(self, obj): return len(obj.points) def index_queryset(self): return Story.objects.filter(status__exact='published', is_template=False) def should_update(self, instance, **kwargs): """ Determine if an object should be updated in the index. """ should_update = True translation_set = getattr(instance, instance.translation_set) if translation_set.count() == 0: should_update = False if 'action' in kwargs: # The signal is m2m_changed. We only want to update # on the post actions if kwargs['action'] in ('pre_add', 'pre_remove', 'pre_clear'): should_update = False return should_update def should_remove_on_update(self, instance, **kwargs): if instance.status != 'published': return True if instance.is_template == True: return True return False def update_object(self, instance, using=None, **kwargs): """ Update the index for a single object. Attached to the class's post-save hook. This version removes unpublished stories from the index """ if self.should_remove_on_update(instance, **kwargs): self.remove_object(instance, using, **kwargs) else: super(StoryIndex, self).update_object(instance, using, **kwargs) def translation_update_object(self, sender, instance, **kwargs): """Signal handler for updating story index when the translation changes""" # Deal with race condition when stories are deleted # See issue #138 try: self.update_object(instance.story) except Story.DoesNotExist: pass def location_update_object(self, sender, instance, **kwargs): """Signal handler for updating story index when a related location changes""" for story in instance.stories.all(): self.update_object(story) def _setup_save(self): super(StoryIndex, self)._setup_save() # Update object when many-to-many fields change signals.m2m_changed.connect( self.update_object, sender=self.get_model().organizations.through) signals.m2m_changed.connect(self.update_object, sender=self.get_model().projects.through) signals.m2m_changed.connect(self.update_object, sender=self.get_model().topics.through) signals.m2m_changed.connect(self.update_object, sender=self.get_model().locations.through) signals.m2m_changed.connect(self.update_object, sender=self.get_model().places.through) signals.post_save.connect(self.translation_update_object, sender=StoryTranslation) signals.post_save.connect(self.location_update_object, sender=Location) def _teardown_save(self): super(StoryIndex, self)._teardown_save() signals.m2m_changed.disconnect( self.update_object, sender=self.get_model().organizations.through) signals.m2m_changed.disconnect( self.update_object, sender=self.get_model().projects.through) signals.m2m_changed.disconnect(self.update_object, sender=self.get_model().topics.through) signals.m2m_changed.disconnect( self.update_object, sender=self.get_model().locations.through) signals.m2m_changed.disconnect(self.update_object, sender=self.get_model().places.through) signals.post_save.disconnect(self.translation_update_object, sender=StoryTranslation) signals.post_save.disconnect(self.location_update_object, sender=Location) def _setup_delete(self): super(StoryIndex, self)._setup_delete() signals.post_delete.connect(self.translation_update_object, sender=StoryTranslation) def _teardown_delete(self): super(StoryIndex, self)._teardown_delete() signals.post_delete.disconnect(self.translation_update_object, sender=StoryTranslation)
class StoryIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) author = indexes.FacetCharField(model_attr='author') published = indexes.FacetDateTimeField(model_attr='published') created = indexes.FacetDateTimeField(model_attr='created') last_edited = indexes.FacetDateTimeField(model_attr='last_edited') # TODO: Use a meta class to dynamically populate these from "official" # tag sets topic_ids = indexes.FacetMultiValueField() organization_ids = indexes.FacetMultiValueField() project_ids = indexes.FacetMultiValueField() language_ids = indexes.FacetMultiValueField() place_ids = indexes.FacetMultiValueField() points = GeoHashMultiValueField() num_points = indexes.IntegerField() suggestions = TextSpellField() def get_model(self): return Story def prepare_topic_ids(self, obj): return [topic.id for topic in obj.topics.all()] def prepare_organization_ids(self, obj): return [ organization.organization_id for organization in obj.organizations.all() ] def prepare_project_ids(self, obj): return [project.project_id for project in obj.projects.all()] def prepare_language_ids(self, obj): return obj.get_languages() def prepare_place_ids(self, obj): return [place.place_id for place in obj.inherited_places] def prepare_points(self, obj): return ["%s,%s" % (point[0], point[1]) for point in obj.points] def prepare_num_points(self, obj): return len(obj.points) def prepare(self, obj): prepared_data = super(StoryIndex, self).prepare(obj) prepared_data['suggestions'] = prepared_data['text'] return prepared_data def index_queryset(self, using=None): """ Get the default QuerySet to index when doing a full update. Excludes unpublish stories, template stories, and connected stories. """ return Story.objects.filter(status__exact='published', is_template=False)\ .exclude(source__relation_type='connected') def should_update(self, instance, **kwargs): """ Determine if an object should be updated in the index. """ should_update = True translation_set = getattr(instance, instance.translation_set) if translation_set.count() == 0: should_update = False if 'action' in kwargs: # The signal is m2m_changed. We only want to update # on the post actions if kwargs['action'] in ('pre_add', 'pre_remove', 'pre_clear'): should_update = False return should_update def should_remove_on_update(self, instance, **kwargs): if instance.status != 'published': return True if instance.is_template == True: return True if instance.is_connected() == True: return True return False def update_object(self, instance, using=None, **kwargs): """ Update the index for a single object. Attached to the class's post-save hook. This version removes unpublished stories from the index """ if self.should_remove_on_update(instance, **kwargs): self.remove_object(instance, using, **kwargs) else: super(StoryIndex, self).update_object(instance, using, **kwargs) def translation_update_object(self, instance, **kwargs): """Signal handler for updating story index when the translation changes""" # Deal with race condition when stories are deleted # See issue #138 try: self.update_object(instance.story) except Story.DoesNotExist: pass def location_update_object(self, instance, **kwargs): """Signal handler for updating story index when a related location changes""" for story in instance.stories.all(): self.update_object(story) def section_translation_update_object(self, instance, **kwargs): """ Signal handler for updating story index when a related section translation changes This is needed because the section titles in all languages are part of the document field of the index. """ self.update_object(instance.section.story) def asset_translation_update_object(self, instance, **kwargs): """ Signal handler for updating story index when a related text asset translation changes This is needed because the text from text assets is part of the document field in the index. """ stories = [] if instance.asset.type == 'text': for section in instance.asset.sections.all(): # Should I use a set here to make this faster? if section.story not in stories: stories.append(section.story) for story in stories: self.update_object(story) def cache_story_for_delete(self, instance, **kwargs): """ Store a reference to the section asset's story This makes the story available to post_delete signal handlers, because it won't neccessarily be available via instance.section.story at that point. This is designed to be attached to the pre_delete signal """ instance._story = instance.section.story def asset_relation_update_object(self, instance, **kwargs): """ Signal handler for when an asset to section relationship is created or destroyed. This is needed because the text from assets is part of the document field of the index. """ if instance.asset.type == 'text': # Try using the cached story. This will be present if # we're deleting the section asset story = getattr(instance, '_story', None) if story is None: # No cached story present, it's safe to get it by following # the relations story = instance.section.story self.update_object(story)
class NewsIndex(indexes.SearchIndex, indexes.Indexable): # common facets; state_name = indexes.FacetMultiValueField() financial_mechanism_ss = indexes.FacetMultiValueField() programme_area_ss = indexes.FacetMultiValueField() priority_sector_ss = indexes.FacetMultiValueField() programme_name = indexes.FacetMultiValueField() programme_status = indexes.FacetMultiValueField() outcome_ss = indexes.FacetMultiValueField() kind = indexes.FacetCharField() # specific facets project_name = indexes.FacetMultiValueField() project_name_auto = indexes.EdgeNgramField() project_status = indexes.FacetMultiValueField() geotarget = indexes.FacetCharField() geotarget_auto = indexes.EdgeNgramField() theme_ss = indexes.FacetMultiValueField() # specific fields text = indexes.CharField(document=True, use_template=True) summary = indexes.CharField(model_attr='summary', indexed=False) name = indexes.CharField(model_attr='title', indexed=False) url = indexes.CharField(model_attr='link', indexed=False) image = indexes.CharField(model_attr='image', indexed=False) created_dt = indexes.DateTimeField(model_attr='created', indexed=False, null=True) def get_model(self): return News def prepare_kind(self, obj): return 'News' def index_queryset(self, using=None): return (self.get_model().objects.select_related( 'project', 'project__financial_mechanism', 'project__outcome', 'project__programme', 'project__programme_area', 'project__programme_area__priority_sector', 'project__state', ).prefetch_related('programmes', 'programmes__outcomes', 'project__themes')) def prepare_state_name(self, obj): if self.project: return [self.project.state.name] elif self.programmes: # Get this from ProgrammeOutcome, because of IN22 return list( set([programme['country'] for programme in self.programmes])) return None def prepare_financial_mechanism_ss(self, obj): if self.project: return [self.project.financial_mechanism.grant_name] if self.programmes: return list( set([programme['mechanism'] for programme in self.programmes])) return None def prepare_programme_area_ss(self, obj): if self.project: return [self.project.programme_area.name] if self.programmes: return list( set([programme['area'] for programme in self.programmes])) return None def prepare_priority_sector_ss(self, obj): if self.project: return [self.project.programme_area.priority_sector.name] if self.programmes.exists(): return list( set([programme['sector'] for programme in self.programmes])) return None def prepare_programme_name(self, obj): if self.project: return [ '{}: {}'.format(self.project.programme.code, self.project.programme.name) ] if self.programmes: return list( set([ '{}: {}'.format(programme['code'], programme['name']) for programme in self.programmes ])) return None def prepare_project_name(self, obj): if self.project: return [ '{}: {}'.format(self.project.code, ' '.join(self.project.name.split())) ] return None def prepare_programme_status(self, obj): if self.project: return [self.project.programme.status] if self.programmes: return list( set([programme['status'] for programme in self.programmes])) return None def prepare_outcome_ss(self, obj): if self.project: return [self.project.outcome.name.strip()] if self.programmes.exists(): return list( set([ programme['outcome_name'] for programme in self.programmes if not programme['outcome_fbl'] ])) return None def prepare_project_status(self, obj): if self.project: return [self.project.status] return None def prepare_geotarget(self, obj): if self.project: if len(self.project.nuts) > 2: return [ '{}: {}, {}'.format(self.project.nuts, self.project.geotarget, STATES[self.project.nuts[:2]]) ] else: return [ '{}: {}'.format(self.project.nuts, self.project.geotarget) ] return None def prepare_theme_ss(self, obj): if self.project: return list( set([theme.name for theme in self.project.themes.all()])) return None def prepare(self, obj): self.project = None self.programmes = list() try: if obj.project: self.project = obj.project except Project.DoesNotExist: pass if obj.programmes.exists(): self.programmes = (obj.programmes.annotate( area=F('programme_areas__name'), mechanism=F( 'programme_areas__financial_mechanism__grant_name'), outcome_name=F('outcomes__outcome__name'), outcome_fbl=F('outcomes__outcome__fixed_budget_line'), sector=F('programme_areas__priority_sector__name'), country=F('outcomes__state__name')).values( 'area', 'code', 'mechanism', 'name', 'outcome_name', 'outcome_fbl', 'sector', 'country', 'status')) self.prepared_data = super().prepare(obj) self.prepared_data['geotarget_auto'] = (' '.join( self.prepared_data['geotarget']) if self.prepared_data['geotarget'] else None) self.prepared_data['project_name_auto'] = ( ' '.join(self.prepared_data['project_name']) if self.prepared_data['project_name'] else None) return self.prepared_data
class ProgrammeIndex(indexes.SearchIndex, indexes.Indexable): # common facets state_name = indexes.FacetMultiValueField() programme_area_ss = indexes.FacetMultiValueField() priority_sector_ss = indexes.FacetMultiValueField() financial_mechanism_ss = indexes.FacetMultiValueField() outcome_ss = indexes.FacetMultiValueField() outcome_ss_auto = indexes.EdgeNgramField() programme_name = indexes.FacetMultiValueField() programme_status = indexes.FacetMultiValueField(model_attr='status') kind = indexes.FacetCharField() # specific facets code = indexes.FacetCharField(model_attr='code') # specific fields text = indexes.CharField(document=True, use_template=True) url = indexes.CharField(model_attr='url', indexed=False, null=True) summary = indexes.CharField(model_attr='summary', indexed=False) name = indexes.CharField(model_attr='name', indexed=False) grant = indexes.DecimalField() def get_model(self): return Programme def index_queryset(self, using=None): return (self.get_model().objects.filter(is_tap=False).prefetch_related( 'outcomes', 'outcomes__state', 'programme_areas', 'programme_areas__priority_sector', )) def prepare_kind(self, obj): return 'Programme' def prepare_state_name(self, obj): # Get this from ProgrammeOutcome, because of IN22 return list( set([outcome['state_name'] for outcome in self.programme_outcomes])) def prepare_programme_name(self, obj): return ['{}: {}'.format(obj.code, ' '.join(obj.name.split()))] def prepare_programme_area_ss(self, obj): return list(set([area['name'] for area in self.programme_areas])) def prepare_priority_sector_ss(self, obj): return list(set([area['sector'] for area in self.programme_areas])) def prepare_financial_mechanism_ss(self, obj): return list(set([area['mechanism'] for area in self.programme_areas])) def prepare_outcome_ss(self, obj): return [ ' '.join(outcome['outcome_name'].split()) for outcome in self.programme_outcomes ] def prepare_grant(self, obj): return obj.allocation_eea + obj.allocation_norway def prepare(self, obj): self.programme_outcomes = (obj.outcomes.exclude( outcome__fixed_budget_line=True).annotate( outcome_name=F('outcome__name'), state_name=F('state__name'), ).values('outcome_name', 'state_name')) self.programme_areas = (obj.programme_areas.annotate( mechanism=F('financial_mechanism__grant_name'), sector=F('priority_sector__name'), ).values('mechanism', 'name', 'sector')) self.prepared_data = super().prepare(obj) self.prepared_data['outcome_ss_auto'] = ( ' '.join(self.prepared_data['outcome_ss']) if self.prepared_data['outcome_ss'] else None) return self.prepared_data
class OrganisationIndex(indexes.SearchIndex, indexes.Indexable): # common facets state_name = indexes.FacetMultiValueField() programme_status = indexes.FacetMultiValueField() financial_mechanism_ss = indexes.FacetMultiValueField() programme_name = indexes.FacetMultiValueField() programme_name_auto = indexes.EdgeNgramField() project_name = indexes.FacetMultiValueField() project_name_auto = indexes.EdgeNgramField() programme_area_ss = indexes.FacetMultiValueField() priority_sector_ss = indexes.FacetMultiValueField() text = indexes.CharField(document=True, use_template=True) kind = indexes.FacetCharField() # specific facets project_status = indexes.FacetMultiValueField() org_type_category = indexes.FacetCharField(model_attr='orgtypecateg') org_type = indexes.FacetCharField(model_attr='orgtype') country = indexes.FacetCharField(model_attr='country') city = indexes.FacetCharField(model_attr='city') city_auto = indexes.EdgeNgramField(model_attr='city') geotarget = indexes.FacetCharField(null=True) geotarget_auto = indexes.EdgeNgramField(null=True) role_ss = indexes.FacetMultiValueField() role_max_priority_code = indexes.IntegerField() # extra data; avoid db hit org_name = indexes.FacetCharField() org_name_auto = indexes.EdgeNgramField() domestic_name = indexes.CharField(model_attr='domestic_name', null=True) # Highest number = max priority for role. Others default to priority 0. ROLE_PRIORITIES = { 'National Focal Point': 7, # NFP 'Programme Operator': 6, # PO 'Donor Programme Partner': 5, # DPP 'Donor Project Partner': 4, # PJDPP 'Programme Partner': 3, # PP 'Project Partner': 2, # PJPP 'Project Promoter': 1, # PJPT } # Caches, lazy init ALL_PROGRAMMES = {} ALL_PROJECTS = {} ALL_ROLES = {} def __init__(self): super().__init__() if not self.ALL_PROGRAMMES: OrganisationIndex.init_caches() @classmethod def init_caches(cls): # Cache programmes _fields = { 'prg_code': F('programme__code'), 'prg_name': F('programme__name'), 'area': F('outcome__programme_area__name'), 'sector': F('outcome__programme_area__priority_sector__name'), 'fm': F('outcome__programme_area__financial_mechanism__grant_name'), 'state_name': F('state__name'), 'status': F('programme__status'), } _all_programmes_raw = (ProgrammeOutcome.objects.all().select_related( 'programme', 'outcome', 'outcome__programe_area__financial_mechanism', 'outcome__programe_area__priority_sector', 'outcome__programe_area', 'state', ).exclude(programme__isnull=True).exclude( programme__is_tap=True).annotate(**_fields).values( *_fields.keys()).distinct()) ALL_PROGRAMMES = defaultdict(lambda: { 'fms': set(), 'sectors': set(), 'areas': set(), 'states': set(), }) for prg in _all_programmes_raw: ALL_PROGRAMMES[prg['prg_code']]['name'] = prg['prg_name'].strip() ALL_PROGRAMMES[prg['prg_code']]['fms'].add(prg['fm']) ALL_PROGRAMMES[prg['prg_code']]['sectors'].add(prg['sector']) ALL_PROGRAMMES[prg['prg_code']]['areas'].add(prg['area']) ALL_PROGRAMMES[prg['prg_code']]['states'].add(prg['state_name']) ALL_PROGRAMMES[prg['prg_code']]['status'] = prg['status'] cls.ALL_PROGRAMMES = ALL_PROGRAMMES # Cache projects _fields = { 'prj_code': F('code'), 'prj_name': F('name'), 'prg_code': F('programme_id'), 'area': F('programme_area__name'), 'sector': F('priority_sector__name'), 'fm': F('financial_mechanism__grant_name'), 'state_name': F('state__name'), 'prj_status': F('status'), } _all_projects_raw = (Project.objects.all().select_related( 'financial_mechanism', 'priority_sector', 'programe_area', 'state', ).annotate(**_fields).values(*_fields.keys())) cls.ALL_PROJECTS = {prj['prj_code']: prj for prj in _all_projects_raw} cls.ALL_ROLES = { x.code: x.role for x in OrganisationRole.objects.all() } def index_queryset(self, using=None): return (self.get_model().objects.prefetch_related('roles')) def get_model(self): return Organisation def prepare_kind(self, obj): return 'Organisation' def prepare_financial_mechanism_ss(self, obj): fm = [self.ALL_PROJECTS[prj]['fm'] for prj in self.projects] for prg in self.programmes: fm += list(self.ALL_PROGRAMMES[prg]['fms']) return list(set(fm)) def prepare_state_name(self, obj): # programme IN22 can have multiple states states = [ self.ALL_PROJECTS[prj]['state_name'] for prj in self.projects ] for prg in self.programmes: states += list(self.ALL_PROGRAMMES[prg]['states']) return list(set(states)) def prepare_programme_status(self, obj): statuses = [ self.ALL_PROGRAMMES[programme]['status'] for programme in self.programmes ] # Add programme status from projects also statuses += [ self.ALL_PROGRAMMES[self.ALL_PROJECTS[prj_code]['prg_code']] ['status'] for prj_code in self.projects ] return list(set(statuses)) def prepare_project_status(self, obj): return list( set([ self.ALL_PROJECTS[project_code]['prj_status'] for project_code in self.projects ])) def prepare_programme_area_ss(self, obj): areas = [self.ALL_PROJECTS[prj]['area'] for prj in self.projects] for prg in self.programmes: areas += list(self.ALL_PROGRAMMES[prg]['areas']) return list(set(areas)) def prepare_priority_sector_ss(self, obj): sectors = [self.ALL_PROJECTS[prj]['sector'] for prj in self.projects] for prg in self.programmes: sectors += list(self.ALL_PROGRAMMES[prg]['sectors']) return list(set(sectors)) def prepare_programme_name(self, obj): prg_codes = list(self.programmes) prg_codes += [ self.ALL_PROJECTS[project_code]['prg_code'] for project_code in self.projects ] return [ '{}: {}'.format( prg_code, ' '.join(self.ALL_PROGRAMMES[prg_code]['name'].split())) for prg_code in set(prg_codes) ] def prepare_project_name(self, obj): return [ '{}: {}'.format( project_code, ' '.join(self.ALL_PROJECTS[project_code]['prj_name'].split())) for project_code in self.projects ] def prepare_role_ss(self, obj): return [self.ALL_ROLES[role_code] for role_code in self.roles] def prepare_geotarget(self, obj): # obj.nuts and obj.geotarget can be empty string if not obj.nuts: if obj.geotarget: return [obj.geotarget] else: return None if len(obj.nuts) > 2: return [ '{}: {}, {}'.format(obj.nuts, obj.geotarget, STATES[obj.nuts[:2]]) ] else: return ['{}: {}'.format(obj.nuts, obj.geotarget)] def prepare_org_name(self, obj): return ' '.join(obj.name.split()) def prepare(self, obj): self.projects = list() self.programmes = list() self.roles = set() for role in obj.roles.all(): if role.project_id and self.ALL_PROJECTS[role.project_id].get( 'prj_code'): # check prj_code because self.ALL_PROJECTS is defaultdict self.projects.append(role.project_id) elif role.programme_id and self.ALL_PROGRAMMES[ role.programme_id].get('name'): # check programme name because self.ALL_PROGRAMMES is defaultdict self.programmes.append(role.programme_id) if role.organisation_role_id != 'DS' and (role.project_id or role.programme_id): # Skip donor states and organisations with no project nor programme self.roles.add(role.organisation_role_id) if len(self.roles) == 0: raise exceptions.SkipDocument self.prepared_data = super().prepare(obj) # Add extra data in text field to avoid extra queries in template self.prepared_data['text'] += ' '.join( self.prepared_data['state_name'] + self.prepared_data['programme_name'] + self.prepared_data['project_name']) self.prepared_data['programme_name_auto'] = ( ' '.join(self.prepared_data['programme_name']) if self.prepared_data['programme_name'] else None) self.prepared_data['project_name_auto'] = ( ' '.join(self.prepared_data['project_name']) if self.prepared_data['project_name'] else None) self.prepared_data['geotarget_auto'] = (' '.join( self.prepared_data['geotarget']) if self.prepared_data['geotarget'] else None) self.prepared_data['org_name_auto'] = self.prepared_data['org_name'] if self.prepared_data['role_ss']: self.prepared_data['role_max_priority_code'] = reduce( lambda max_value, role: max(max_value, self.ROLE_PRIORITIES.get(role, 0)), self.prepared_data['role_ss'], 0) else: self.prepared_data['role_max_priority_code'] = None return self.prepared_data
class ProjectIndex(indexes.SearchIndex, indexes.Indexable): # common facets state_name = indexes.FacetMultiValueField(model_attr='state__name') financial_mechanism_ss = indexes.FacetMultiValueField() programme_area_ss = indexes.FacetMultiValueField() priority_sector_ss = indexes.FacetMultiValueField() programme_name = indexes.FacetMultiValueField() programme_status = indexes.FacetMultiValueField( model_attr='programme__status') outcome_ss = indexes.FacetMultiValueField() outcome_ss_auto = indexes.EdgeNgramField() kind = indexes.FacetCharField() # specific facets code = indexes.FacetCharField(model_attr='code') project_status = indexes.FacetMultiValueField(model_attr='status') geotarget = indexes.FacetCharField(model_attr='geotarget') geotarget_auto = indexes.EdgeNgramField(model_attr='geotarget') theme_ss = indexes.FacetMultiValueField() # specific fields text = indexes.CharField(document=True, use_template=True) summary = indexes.CharField(model_attr='summary', indexed=False) url = indexes.CharField(model_attr='url', indexed=False, null=True) name = indexes.CharField(model_attr='name', indexed=False) grant = indexes.DecimalField(model_attr='allocation') def index_queryset(self, using=None): return ( self.get_model().objects.select_related( 'financial_mechanism', 'outcome', 'programme', 'programme_area', 'programme_area__priority_sector', 'state', ).prefetch_related('themes') # using prefetch_related may require --batch-size 999 to avoid # sqlite3.OperationalError: too many SQL variables ) def get_model(self): return Project def prepare_kind(self, obj): return 'Project' def prepare_financial_mechanism_ss(self, obj): return [obj.financial_mechanism.grant_name] def prepare_programme_area_ss(self, obj): return [obj.programme_area.name] def prepare_priority_sector_ss(self, obj): return [obj.programme_area.priority_sector.name] def prepare_programme_name(self, obj): return [ '{}: {}'.format(obj.programme.code, ' '.join(obj.programme.name.split())) ] def prepare_outcome_ss(self, obj): return [' '.join(obj.outcome.name.split())] def prepare_geotarget(self, obj): if len(obj.nuts) > 2: return [ '{}: {}, {}'.format(obj.nuts, obj.geotarget, STATES[obj.nuts[:2]]) ] else: return ['{}: {}'.format(obj.nuts, obj.geotarget)] def prepare_theme_ss(self, obj): return list(set([theme.name for theme in obj.themes.all()])) def prepare(self, obj): self.prepared_data = super().prepare(obj) self.prepared_data['outcome_ss_auto'] = ( ' '.join(self.prepared_data['outcome_ss']) if self.prepared_data['outcome_ss'] else None) self.prepared_data['geotarget_auto'] = (' '.join( self.prepared_data['geotarget']) if self.prepared_data['geotarget'] else None) return self.prepared_data