class DatasetIndex(indexes.ModelSearchIndex, indexes.Indexable):
title = indexes.CharField(model_attr='title', boost=1.25)
description = indexes.CharField(model_attr='description', boost=1.125)
group_name = indexes.CharField(model_attr='group_name', faceted=True)
access_type = indexes.CharField(model_attr='access_type', faceted=True)
states = indexes.FacetMultiValueField(model_attr='states')
division_names = indexes.FacetMultiValueField(model_attr='division_names')
sectors = indexes.FacetMultiValueField(model_attr='sectors')
formats = indexes.MultiValueField(model_attr='formats', boost=0.6)
tags = indexes.MultiValueField(model_attr='tags', boost=0.8)
class Meta:
model = Dataset
excludes = ['uuid', 'created_at', 'updated_at', 'formats', 'tags']
def prepare_states(self, object):
states_list = list(object.states_expanded())
return states_list
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
return self.get_model().objects.filter(updated_at__lte=timezone.now())
def get_updated_field(self):
return 'updated_at'
class PublicBodyIndex(SearchIndex, indexes.Indexable):
text = indexes.EdgeNgramField(document=True, use_template=True)
name = indexes.CharField(model_attr='name', boost=1.5)
name_auto = SuggestField(model_attr='all_names', boost=2.5)
jurisdiction = indexes.FacetCharField(model_attr='jurisdiction__name',
default='')
classification = indexes.FacetMultiValueField()
categories = indexes.FacetMultiValueField()
def get_model(self):
return PublicBody
def index_queryset(self, **kwargs):
"""Used when the entire index for model is updated."""
return self.get_model().objects.get_for_search_index()
def prepare_classification(self, obj):
if obj.classification is None:
return []
return [obj.classification.name
] + [c.name for c in obj.classification.get_ancestors()]
def prepare_categories(self, obj):
cats = obj.categories.all()
return [o.name for o in cats
] + [c.name for o in cats for c in o.get_ancestors()]
class InitiativeIndex(AbstractIndex, indexes.ModelSearchIndex,
indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
# facets
years = indexes.FacetMultiValueField()
recipients = indexes.FacetMultiValueField()
agency = indexes.FacetCharField()
aid_types = indexes.FacetMultiValueField()
channels = indexes.FacetMultiValueField()
finance_type = indexes.FacetCharField()
sectors = indexes.FacetMultiValueField()
def prepare_years(self, obj):
return obj.years_range()
def prepare_recipients(self, obj):
return self._prepare_codelist(obj.recipients(), roots=False)
def prepare_agency(self, obj):
agency = obj.agency()
return agency.code if agency else None
def prepare_aid_types(self, obj):
return self._prepare_codelist(obj.aid_types())
def prepare_channels(self, obj):
return self._prepare_codelist(obj.channels())
def prepare_finance_type(self, obj):
finance_type = obj.finance_type()
return finance_type.code if finance_type else None
def prepare_sectors(self, obj):
return self._prepare_codelist(obj.sectors())
class Meta:
model = Initiative
# fields = ['years', 'recipients', 'agency', 'aid_types',
# 'channels', 'finance_type', 'sectors',
# 'code', 'start_year', 'end_year', 'has_focus']
excludes = [
'code', 'title', 'total_project_costs', 'loan_amount_approved',
'grant_amount_approved', 'photo_set', 'document_set', 'updated_at',
'created_at'
]
for f in Initiative._meta.fields:
if f.attname.endswith('_temp') or f.attname[-3:] in ('_it', '_en'):
excludes.append(f.attname)
class EventIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
categories = indexes.FacetMultiValueField()
title = indexes.CharField(model_attr='title')
end_date = indexes.DateField(
model_attr='end_date',
default=lambda: datetime.now() + timedelta(days=(30 * 365)))
# If no expiry, expire after 30 years
city = indexes.CharField(model_attr='venue__city', faceted=True, null=True)
neighborhood = indexes.CharField(model_attr='venue__neighborhood',
faceted=True,
null=True)
point = indexes.LocationField(model_attr='venue__point', null=True)
# We add this for autocomplete.
title_auto = indexes.EdgeNgramField(use_template=True)
def get_model(self):
return Event
def prepare_categories(self, obj):
return [category.name for category in obj.category.all()]
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
return self.get_model().objects.filter(publish=True)
class PublicationIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.NgramField(document=True, use_template=True)
title = indexes.NgramField(model_attr='titre')
bureaux = indexes.FacetMultiValueField(null=True, stored=True)
annee = indexes.FacetField(stored=True, null=True)
section = indexes.FacetField(stored=True, null=True)
date_pub = indexes.DateField(model_attr='date_pub', null=True)
def prepare_bureaux(self, obj):
try:
return [b.nom for b in obj.bureau.all()]
except ObjectDoesNotExist as e:
print(e)
return [u'Non précisé']
def prepare_annee(self, obj):
if obj.date_pub is not None:
return str(obj.date_pub.year)
def get_model(self):
return Publication
def prepare_section(self, obj):
return u"Publication"
def index_queryset(self, using=None):
return Publication.objects.filter(
status__in=[3, 5, 6]).order_by("date_pub")
class PublicBodyIndex(SearchIndex, indexes.Indexable):
text = indexes.EdgeNgramField(document=True, use_template=True)
name = indexes.CharField(model_attr='name', boost=1.5)
name_auto = indexes.NgramField(model_attr='name')
jurisdiction = indexes.FacetCharField(model_attr='jurisdiction__name',
default='')
tags = indexes.FacetMultiValueField()
url = indexes.CharField(model_attr='get_absolute_url')
def get_model(self):
return PublicBody
def index_queryset(self, **kwargs):
"""Used when the entire index for model is updated."""
return self.get_model().objects.get_for_search_index()
def prepare_tags(self, obj):
return [t.name for t in obj.tags.all()]
def prepare(self, obj):
data = super(PublicBodyIndex, self).prepare(obj)
if obj.classification in PUBLIC_BODY_BOOSTS:
data['boost'] = PUBLIC_BODY_BOOSTS[obj.classification]
print("Boosting %s at %f" % (obj, data['boost']))
return data
class CandidateIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
first_name = indexes.CharField(indexed=True)
party = indexes.FacetCharField(indexed=True)
college = indexes.FacetCharField(indexed=True)
state = indexes.FacetCharField(indexed=True)
issues = indexes.FacetMultiValueField(indexed=True)
race = indexes.FacetCharField(indexed=True)
race_type = indexes.FacetCharField(indexed=True)
women = indexes.BooleanField(indexed=True)
active = indexes.BooleanField(indexed=True)
random = indexes.CharField()
def get_model(self):
return Candidate
def prepare_first_name(self, obj):
return obj.first_name
def prepare_issues(self, obj):
return [(issue.issue.parent_name)
for issue in obj.issues.all()] or None
def prepare_party(self, obj):
return obj.party
def prepare_state(self, obj):
return obj.state
def prepare_race(self, obj):
if obj.race:
return obj.race.title
def prepare_active(self, obj):
return obj.active
def prepare_women(self, obj):
if obj.man:
return False
return True
def prepare_race_type(self, obj):
if obj.race:
return obj.race.race_type
def prepare_random(self, obj):
return random.randint(0, 99999999)
def prepare_college(self, obj):
if obj.college:
return obj.college.name
return None
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
return self.get_model().objects.all()
class OrganizationIndex(CelerySearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
created = indexes.DateTimeField(model_attr='created')
modified = indexes.DateTimeField(model_attr='modified')
state = indexes.FacetCharField(model_attr='state')
tags = indexes.FacetMultiValueField()
def prepare_tags(self, my_model):
return [tag.id for tag in my_model.tags.all()]
def get_model(self):
return models.Organization
class AufIndex(AldrynIndexBase):
index_title = True
text = indexes.NgramField(document=True, use_template=False)
title = indexes.NgramField(stored=True, indexed=False)
description = indexes.NgramField(indexed=False, stored=True)
bureaux = indexes.FacetMultiValueField(stored=True, null=True)
annee = indexes.FacetField(stored=True, null=True)
def prepare_bureaux(self, obj):
try:
return [b.nom for b in obj.page.bureauextension.bureau.all()]
except ObjectDoesNotExist, e:
for path in BUREAU_SLUGS.keys():
if path in obj.page.get_absolute_url():
return [BUREAU_SLUGS[path]]
return [u'Non précisé']
class ProjectIndex(AbstractIndex, indexes.ModelSearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
# facets
years = indexes.FacetMultiValueField()
# facets for code-lists
recipient = indexes.FacetCharField()
agencies = indexes.FacetMultiValueField()
aid_types = indexes.FacetMultiValueField()
channels = indexes.FacetMultiValueField()
finance_types = indexes.FacetMultiValueField()
sectors = indexes.FacetMultiValueField()
def prepare_recipient(self, obj):
return obj.recipient.code
def prepare_agencies(self, obj):
return self._prepare_codelist(obj.agencies(), roots=False)
def prepare_aid_types(self, obj):
return self._prepare_codelist(obj.aid_types())
def prepare_channels(self, obj):
return self._prepare_codelist(obj.channels())
def prepare_finance_types(self, obj):
return self._prepare_codelist(obj.finance_types())
def prepare_sectors(self, obj):
return self._prepare_codelist(obj.sectors())
def get_facets_counts(self):
return self.objects.facet('year').facet_counts().get('fields', {})
class Meta:
model = Project
# fields = ['years', 'markers', 'recipient', 'agency', 'aid_type',
# 'channel', 'finance_type', 'sector', 'initiative',
# 'crsid', 'start_year', 'end_year', 'has_focus']
excludes = [
'is_suspended', 'status', 'expected_start_year',
'expected_completion_year', 'expected_start_date',
'completion_date', 'last_update', 'outcome', 'beneficiaries',
'beneficiaries_female', 'total_project_costs', 'other_financiers',
'loan_amount_approved', 'grant_amount_approved',
'counterpart_authority', 'email', 'location', 'description',
'title', 'title_it', 'title_en', 'description_it',
'description_en', 'number', 'outcome_it', 'beneficiaries_it',
'other_financiers_it', 'counterpart_authority_it', 'location_it',
'outcome_en', 'beneficiaries_en', 'other_financiers_en',
'counterpart_authority_en', 'location_en'
]
class AufIndex(indexes.SearchIndex):
text = indexes.NgramField(document=True, use_template=True)
title = indexes.NgramField(model_attr='titre')
bureaux = indexes.FacetMultiValueField(null=True, stored=True)
annee = indexes.FacetField(stored=True, null=True)
section = indexes.FacetField(stored=True, null=True)
partenaire = indexes.FacetField(stored=True, null=True)
date_pub = indexes.DateField(model_attr='date_pub', null=True)
def prepare_bureaux(self, obj):
regions = []
try:
if obj.bureau.all(
).count == 0 or obj.status == "3" or obj.status == "5":
regions.append(u'International')
return regions + [b.nom for b in obj.bureau.all()]
except ObjectDoesNotExist as e:
print(e)
return []
def prepare_annee(self, obj):
if obj.date_pub is not None:
return str(obj.date_pub.year)
class QuestionIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
content_auto = indexes.EdgeNgramField(model_attr='question_text')
state = indexes.IntegerField(model_attr='state')
region = indexes.MultiValueField()
organisation = indexes.CharField(model_attr='organisation__id')
#Facets
state_facet = indexes.FacetIntegerField(model_attr = 'state')
institution_facet = indexes.FacetMultiValueField()
promotor_facet = indexes.FacetMultiValueField()
faculty_facet = indexes.FacetMultiValueField()
education_facet = indexes.FacetMultiValueField()
subject_facet = indexes.FacetMultiValueField()
key_word_facet = indexes.FacetMultiValueField()
def prepare_region(self, obj):
return [region.region for region in obj.region.all()]
def prepare_institution_facet(self,obj):
return [institution.name for institution in obj.institution.all()]
def prepare_promotor_facet(self,obj):
return ['{0} {1}'.format(promotor.first_name, promotor.last_name) for promotor in obj.promotor.all()]
def prepare_faculty_facet(self,obj):
return [faculty.name for faculty in obj.faculty.all()]
def prepare_education_facet(self, obj):
return [education.education for education in obj.education.all()]
def prepare_subject_facet(self,obj):
return [subject.subject for subject in obj.question_subject.all()]
def prepare_key_word_facet(self,obj):
return [keyword.key_word for keyword in obj.keyword.all()]
def get_model(self):
return Question
class StoryIndex(indexes.RealTimeSearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
author = indexes.FacetCharField(model_attr='author')
published = indexes.FacetDateTimeField(model_attr='published')
created = indexes.FacetDateTimeField(model_attr='created')
last_edited = indexes.FacetDateTimeField(model_attr='last_edited')
# TODO: Use a meta class to dynamically populate these from "official"
# tag sets
topic_ids = indexes.FacetMultiValueField()
organization_ids = indexes.FacetMultiValueField()
project_ids = indexes.FacetMultiValueField()
language_ids = indexes.FacetMultiValueField()
place_ids = indexes.FacetMultiValueField()
points = GeoHashMultiValueField()
num_points = indexes.IntegerField()
def get_model(self):
return Story
def prepare_topic_ids(self, obj):
return [topic.id for topic in obj.topics.all()]
def prepare_organization_ids(self, obj):
return [
organization.organization_id
for organization in obj.organizations.all()
]
def prepare_project_ids(self, obj):
return [project.project_id for project in obj.projects.all()]
def prepare_language_ids(self, obj):
return obj.get_languages()
def prepare_place_ids(self, obj):
return [place.place_id for place in obj.inherited_places]
def prepare_points(self, obj):
return ["%s,%s" % (point[0], point[1]) for point in obj.points]
def prepare_num_points(self, obj):
return len(obj.points)
def index_queryset(self):
return Story.objects.filter(status__exact='published',
is_template=False)
def should_update(self, instance, **kwargs):
"""
Determine if an object should be updated in the index.
"""
should_update = True
translation_set = getattr(instance, instance.translation_set)
if translation_set.count() == 0:
should_update = False
if 'action' in kwargs:
# The signal is m2m_changed. We only want to update
# on the post actions
if kwargs['action'] in ('pre_add', 'pre_remove', 'pre_clear'):
should_update = False
return should_update
def should_remove_on_update(self, instance, **kwargs):
if instance.status != 'published':
return True
if instance.is_template == True:
return True
return False
def update_object(self, instance, using=None, **kwargs):
"""
Update the index for a single object. Attached to the class's
post-save hook.
This version removes unpublished stories from the index
"""
if self.should_remove_on_update(instance, **kwargs):
self.remove_object(instance, using, **kwargs)
else:
super(StoryIndex, self).update_object(instance, using, **kwargs)
def translation_update_object(self, sender, instance, **kwargs):
"""Signal handler for updating story index when the translation changes"""
# Deal with race condition when stories are deleted
# See issue #138
try:
self.update_object(instance.story)
except Story.DoesNotExist:
pass
def location_update_object(self, sender, instance, **kwargs):
"""Signal handler for updating story index when a related location changes"""
for story in instance.stories.all():
self.update_object(story)
def _setup_save(self):
super(StoryIndex, self)._setup_save()
# Update object when many-to-many fields change
signals.m2m_changed.connect(
self.update_object, sender=self.get_model().organizations.through)
signals.m2m_changed.connect(self.update_object,
sender=self.get_model().projects.through)
signals.m2m_changed.connect(self.update_object,
sender=self.get_model().topics.through)
signals.m2m_changed.connect(self.update_object,
sender=self.get_model().locations.through)
signals.m2m_changed.connect(self.update_object,
sender=self.get_model().places.through)
signals.post_save.connect(self.translation_update_object,
sender=StoryTranslation)
signals.post_save.connect(self.location_update_object, sender=Location)
def _teardown_save(self):
super(StoryIndex, self)._teardown_save()
signals.m2m_changed.disconnect(
self.update_object, sender=self.get_model().organizations.through)
signals.m2m_changed.disconnect(
self.update_object, sender=self.get_model().projects.through)
signals.m2m_changed.disconnect(self.update_object,
sender=self.get_model().topics.through)
signals.m2m_changed.disconnect(
self.update_object, sender=self.get_model().locations.through)
signals.m2m_changed.disconnect(self.update_object,
sender=self.get_model().places.through)
signals.post_save.disconnect(self.translation_update_object,
sender=StoryTranslation)
signals.post_save.disconnect(self.location_update_object,
sender=Location)
def _setup_delete(self):
super(StoryIndex, self)._setup_delete()
signals.post_delete.connect(self.translation_update_object,
sender=StoryTranslation)
def _teardown_delete(self):
super(StoryIndex, self)._teardown_delete()
signals.post_delete.disconnect(self.translation_update_object,
sender=StoryTranslation)
class StoryIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
author = indexes.FacetCharField(model_attr='author')
published = indexes.FacetDateTimeField(model_attr='published')
created = indexes.FacetDateTimeField(model_attr='created')
last_edited = indexes.FacetDateTimeField(model_attr='last_edited')
# TODO: Use a meta class to dynamically populate these from "official"
# tag sets
topic_ids = indexes.FacetMultiValueField()
organization_ids = indexes.FacetMultiValueField()
project_ids = indexes.FacetMultiValueField()
language_ids = indexes.FacetMultiValueField()
place_ids = indexes.FacetMultiValueField()
points = GeoHashMultiValueField()
num_points = indexes.IntegerField()
suggestions = TextSpellField()
def get_model(self):
return Story
def prepare_topic_ids(self, obj):
return [topic.id for topic in obj.topics.all()]
def prepare_organization_ids(self, obj):
return [
organization.organization_id
for organization in obj.organizations.all()
]
def prepare_project_ids(self, obj):
return [project.project_id for project in obj.projects.all()]
def prepare_language_ids(self, obj):
return obj.get_languages()
def prepare_place_ids(self, obj):
return [place.place_id for place in obj.inherited_places]
def prepare_points(self, obj):
return ["%s,%s" % (point[0], point[1]) for point in obj.points]
def prepare_num_points(self, obj):
return len(obj.points)
def prepare(self, obj):
prepared_data = super(StoryIndex, self).prepare(obj)
prepared_data['suggestions'] = prepared_data['text']
return prepared_data
def index_queryset(self, using=None):
"""
Get the default QuerySet to index when doing a full update.
Excludes unpublish stories, template stories, and connected stories.
"""
return Story.objects.filter(status__exact='published',
is_template=False)\
.exclude(source__relation_type='connected')
def should_update(self, instance, **kwargs):
"""
Determine if an object should be updated in the index.
"""
should_update = True
translation_set = getattr(instance, instance.translation_set)
if translation_set.count() == 0:
should_update = False
if 'action' in kwargs:
# The signal is m2m_changed. We only want to update
# on the post actions
if kwargs['action'] in ('pre_add', 'pre_remove', 'pre_clear'):
should_update = False
return should_update
def should_remove_on_update(self, instance, **kwargs):
if instance.status != 'published':
return True
if instance.is_template == True:
return True
if instance.is_connected() == True:
return True
return False
def update_object(self, instance, using=None, **kwargs):
"""
Update the index for a single object. Attached to the class's
post-save hook.
This version removes unpublished stories from the index
"""
if self.should_remove_on_update(instance, **kwargs):
self.remove_object(instance, using, **kwargs)
else:
super(StoryIndex, self).update_object(instance, using, **kwargs)
def translation_update_object(self, instance, **kwargs):
"""Signal handler for updating story index when the translation changes"""
# Deal with race condition when stories are deleted
# See issue #138
try:
self.update_object(instance.story)
except Story.DoesNotExist:
pass
def location_update_object(self, instance, **kwargs):
"""Signal handler for updating story index when a related location changes"""
for story in instance.stories.all():
self.update_object(story)
def section_translation_update_object(self, instance, **kwargs):
"""
Signal handler for updating story index when a related section
translation changes
This is needed because the section titles in all languages are part
of the document field of the index.
"""
self.update_object(instance.section.story)
def asset_translation_update_object(self, instance, **kwargs):
"""
Signal handler for updating story index when a related text asset
translation changes
This is needed because the text from text assets is part of the
document field in the index.
"""
stories = []
if instance.asset.type == 'text':
for section in instance.asset.sections.all():
# Should I use a set here to make this faster?
if section.story not in stories:
stories.append(section.story)
for story in stories:
self.update_object(story)
def cache_story_for_delete(self, instance, **kwargs):
"""
Store a reference to the section asset's story
This makes the story available to post_delete signal handlers, because
it won't neccessarily be available via instance.section.story at
that point.
This is designed to be attached to the pre_delete signal
"""
instance._story = instance.section.story
def asset_relation_update_object(self, instance, **kwargs):
"""
Signal handler for when an asset to section relationship is
created or destroyed.
This is needed because the text from assets is part of the
document field of the index.
"""
if instance.asset.type == 'text':
# Try using the cached story. This will be present if
# we're deleting the section asset
story = getattr(instance, '_story', None)
if story is None:
# No cached story present, it's safe to get it by following
# the relations
story = instance.section.story
self.update_object(story)
class NewsIndex(indexes.SearchIndex, indexes.Indexable):
# common facets;
state_name = indexes.FacetMultiValueField()
financial_mechanism_ss = indexes.FacetMultiValueField()
programme_area_ss = indexes.FacetMultiValueField()
priority_sector_ss = indexes.FacetMultiValueField()
programme_name = indexes.FacetMultiValueField()
programme_status = indexes.FacetMultiValueField()
outcome_ss = indexes.FacetMultiValueField()
kind = indexes.FacetCharField()
# specific facets
project_name = indexes.FacetMultiValueField()
project_name_auto = indexes.EdgeNgramField()
project_status = indexes.FacetMultiValueField()
geotarget = indexes.FacetCharField()
geotarget_auto = indexes.EdgeNgramField()
theme_ss = indexes.FacetMultiValueField()
# specific fields
text = indexes.CharField(document=True, use_template=True)
summary = indexes.CharField(model_attr='summary', indexed=False)
name = indexes.CharField(model_attr='title', indexed=False)
url = indexes.CharField(model_attr='link', indexed=False)
image = indexes.CharField(model_attr='image', indexed=False)
created_dt = indexes.DateTimeField(model_attr='created',
indexed=False,
null=True)
def get_model(self):
return News
def prepare_kind(self, obj):
return 'News'
def index_queryset(self, using=None):
return (self.get_model().objects.select_related(
'project',
'project__financial_mechanism',
'project__outcome',
'project__programme',
'project__programme_area',
'project__programme_area__priority_sector',
'project__state',
).prefetch_related('programmes', 'programmes__outcomes',
'project__themes'))
def prepare_state_name(self, obj):
if self.project:
return [self.project.state.name]
elif self.programmes:
# Get this from ProgrammeOutcome, because of IN22
return list(
set([programme['country'] for programme in self.programmes]))
return None
def prepare_financial_mechanism_ss(self, obj):
if self.project:
return [self.project.financial_mechanism.grant_name]
if self.programmes:
return list(
set([programme['mechanism'] for programme in self.programmes]))
return None
def prepare_programme_area_ss(self, obj):
if self.project:
return [self.project.programme_area.name]
if self.programmes:
return list(
set([programme['area'] for programme in self.programmes]))
return None
def prepare_priority_sector_ss(self, obj):
if self.project:
return [self.project.programme_area.priority_sector.name]
if self.programmes.exists():
return list(
set([programme['sector'] for programme in self.programmes]))
return None
def prepare_programme_name(self, obj):
if self.project:
return [
'{}: {}'.format(self.project.programme.code,
self.project.programme.name)
]
if self.programmes:
return list(
set([
'{}: {}'.format(programme['code'], programme['name'])
for programme in self.programmes
]))
return None
def prepare_project_name(self, obj):
if self.project:
return [
'{}: {}'.format(self.project.code,
' '.join(self.project.name.split()))
]
return None
def prepare_programme_status(self, obj):
if self.project:
return [self.project.programme.status]
if self.programmes:
return list(
set([programme['status'] for programme in self.programmes]))
return None
def prepare_outcome_ss(self, obj):
if self.project:
return [self.project.outcome.name.strip()]
if self.programmes.exists():
return list(
set([
programme['outcome_name'] for programme in self.programmes
if not programme['outcome_fbl']
]))
return None
def prepare_project_status(self, obj):
if self.project:
return [self.project.status]
return None
def prepare_geotarget(self, obj):
if self.project:
if len(self.project.nuts) > 2:
return [
'{}: {}, {}'.format(self.project.nuts,
self.project.geotarget,
STATES[self.project.nuts[:2]])
]
else:
return [
'{}: {}'.format(self.project.nuts, self.project.geotarget)
]
return None
def prepare_theme_ss(self, obj):
if self.project:
return list(
set([theme.name for theme in self.project.themes.all()]))
return None
def prepare(self, obj):
self.project = None
self.programmes = list()
try:
if obj.project:
self.project = obj.project
except Project.DoesNotExist:
pass
if obj.programmes.exists():
self.programmes = (obj.programmes.annotate(
area=F('programme_areas__name'),
mechanism=F(
'programme_areas__financial_mechanism__grant_name'),
outcome_name=F('outcomes__outcome__name'),
outcome_fbl=F('outcomes__outcome__fixed_budget_line'),
sector=F('programme_areas__priority_sector__name'),
country=F('outcomes__state__name')).values(
'area', 'code', 'mechanism', 'name', 'outcome_name',
'outcome_fbl', 'sector', 'country', 'status'))
self.prepared_data = super().prepare(obj)
self.prepared_data['geotarget_auto'] = (' '.join(
self.prepared_data['geotarget']) if self.prepared_data['geotarget']
else None)
self.prepared_data['project_name_auto'] = (
' '.join(self.prepared_data['project_name'])
if self.prepared_data['project_name'] else None)
return self.prepared_data
class ProgrammeIndex(indexes.SearchIndex, indexes.Indexable):
# common facets
state_name = indexes.FacetMultiValueField()
programme_area_ss = indexes.FacetMultiValueField()
priority_sector_ss = indexes.FacetMultiValueField()
financial_mechanism_ss = indexes.FacetMultiValueField()
outcome_ss = indexes.FacetMultiValueField()
outcome_ss_auto = indexes.EdgeNgramField()
programme_name = indexes.FacetMultiValueField()
programme_status = indexes.FacetMultiValueField(model_attr='status')
kind = indexes.FacetCharField()
# specific facets
code = indexes.FacetCharField(model_attr='code')
# specific fields
text = indexes.CharField(document=True, use_template=True)
url = indexes.CharField(model_attr='url', indexed=False, null=True)
summary = indexes.CharField(model_attr='summary', indexed=False)
name = indexes.CharField(model_attr='name', indexed=False)
grant = indexes.DecimalField()
def get_model(self):
return Programme
def index_queryset(self, using=None):
return (self.get_model().objects.filter(is_tap=False).prefetch_related(
'outcomes',
'outcomes__state',
'programme_areas',
'programme_areas__priority_sector',
))
def prepare_kind(self, obj):
return 'Programme'
def prepare_state_name(self, obj):
# Get this from ProgrammeOutcome, because of IN22
return list(
set([outcome['state_name']
for outcome in self.programme_outcomes]))
def prepare_programme_name(self, obj):
return ['{}: {}'.format(obj.code, ' '.join(obj.name.split()))]
def prepare_programme_area_ss(self, obj):
return list(set([area['name'] for area in self.programme_areas]))
def prepare_priority_sector_ss(self, obj):
return list(set([area['sector'] for area in self.programme_areas]))
def prepare_financial_mechanism_ss(self, obj):
return list(set([area['mechanism'] for area in self.programme_areas]))
def prepare_outcome_ss(self, obj):
return [
' '.join(outcome['outcome_name'].split())
for outcome in self.programme_outcomes
]
def prepare_grant(self, obj):
return obj.allocation_eea + obj.allocation_norway
def prepare(self, obj):
self.programme_outcomes = (obj.outcomes.exclude(
outcome__fixed_budget_line=True).annotate(
outcome_name=F('outcome__name'),
state_name=F('state__name'),
).values('outcome_name', 'state_name'))
self.programme_areas = (obj.programme_areas.annotate(
mechanism=F('financial_mechanism__grant_name'),
sector=F('priority_sector__name'),
).values('mechanism', 'name', 'sector'))
self.prepared_data = super().prepare(obj)
self.prepared_data['outcome_ss_auto'] = (
' '.join(self.prepared_data['outcome_ss'])
if self.prepared_data['outcome_ss'] else None)
return self.prepared_data
class OrganisationIndex(indexes.SearchIndex, indexes.Indexable):
# common facets
state_name = indexes.FacetMultiValueField()
programme_status = indexes.FacetMultiValueField()
financial_mechanism_ss = indexes.FacetMultiValueField()
programme_name = indexes.FacetMultiValueField()
programme_name_auto = indexes.EdgeNgramField()
project_name = indexes.FacetMultiValueField()
project_name_auto = indexes.EdgeNgramField()
programme_area_ss = indexes.FacetMultiValueField()
priority_sector_ss = indexes.FacetMultiValueField()
text = indexes.CharField(document=True, use_template=True)
kind = indexes.FacetCharField()
# specific facets
project_status = indexes.FacetMultiValueField()
org_type_category = indexes.FacetCharField(model_attr='orgtypecateg')
org_type = indexes.FacetCharField(model_attr='orgtype')
country = indexes.FacetCharField(model_attr='country')
city = indexes.FacetCharField(model_attr='city')
city_auto = indexes.EdgeNgramField(model_attr='city')
geotarget = indexes.FacetCharField(null=True)
geotarget_auto = indexes.EdgeNgramField(null=True)
role_ss = indexes.FacetMultiValueField()
role_max_priority_code = indexes.IntegerField()
# extra data; avoid db hit
org_name = indexes.FacetCharField()
org_name_auto = indexes.EdgeNgramField()
domestic_name = indexes.CharField(model_attr='domestic_name', null=True)
# Highest number = max priority for role. Others default to priority 0.
ROLE_PRIORITIES = {
'National Focal Point': 7, # NFP
'Programme Operator': 6, # PO
'Donor Programme Partner': 5, # DPP
'Donor Project Partner': 4, # PJDPP
'Programme Partner': 3, # PP
'Project Partner': 2, # PJPP
'Project Promoter': 1, # PJPT
}
# Caches, lazy init
ALL_PROGRAMMES = {}
ALL_PROJECTS = {}
ALL_ROLES = {}
def __init__(self):
super().__init__()
if not self.ALL_PROGRAMMES:
OrganisationIndex.init_caches()
@classmethod
def init_caches(cls):
# Cache programmes
_fields = {
'prg_code': F('programme__code'),
'prg_name': F('programme__name'),
'area': F('outcome__programme_area__name'),
'sector': F('outcome__programme_area__priority_sector__name'),
'fm':
F('outcome__programme_area__financial_mechanism__grant_name'),
'state_name': F('state__name'),
'status': F('programme__status'),
}
_all_programmes_raw = (ProgrammeOutcome.objects.all().select_related(
'programme',
'outcome',
'outcome__programe_area__financial_mechanism',
'outcome__programe_area__priority_sector',
'outcome__programe_area',
'state',
).exclude(programme__isnull=True).exclude(
programme__is_tap=True).annotate(**_fields).values(
*_fields.keys()).distinct())
ALL_PROGRAMMES = defaultdict(lambda: {
'fms': set(),
'sectors': set(),
'areas': set(),
'states': set(),
})
for prg in _all_programmes_raw:
ALL_PROGRAMMES[prg['prg_code']]['name'] = prg['prg_name'].strip()
ALL_PROGRAMMES[prg['prg_code']]['fms'].add(prg['fm'])
ALL_PROGRAMMES[prg['prg_code']]['sectors'].add(prg['sector'])
ALL_PROGRAMMES[prg['prg_code']]['areas'].add(prg['area'])
ALL_PROGRAMMES[prg['prg_code']]['states'].add(prg['state_name'])
ALL_PROGRAMMES[prg['prg_code']]['status'] = prg['status']
cls.ALL_PROGRAMMES = ALL_PROGRAMMES
# Cache projects
_fields = {
'prj_code': F('code'),
'prj_name': F('name'),
'prg_code': F('programme_id'),
'area': F('programme_area__name'),
'sector': F('priority_sector__name'),
'fm': F('financial_mechanism__grant_name'),
'state_name': F('state__name'),
'prj_status': F('status'),
}
_all_projects_raw = (Project.objects.all().select_related(
'financial_mechanism',
'priority_sector',
'programe_area',
'state',
).annotate(**_fields).values(*_fields.keys()))
cls.ALL_PROJECTS = {prj['prj_code']: prj for prj in _all_projects_raw}
cls.ALL_ROLES = {
x.code: x.role
for x in OrganisationRole.objects.all()
}
def index_queryset(self, using=None):
return (self.get_model().objects.prefetch_related('roles'))
def get_model(self):
return Organisation
def prepare_kind(self, obj):
return 'Organisation'
def prepare_financial_mechanism_ss(self, obj):
fm = [self.ALL_PROJECTS[prj]['fm'] for prj in self.projects]
for prg in self.programmes:
fm += list(self.ALL_PROGRAMMES[prg]['fms'])
return list(set(fm))
def prepare_state_name(self, obj):
# programme IN22 can have multiple states
states = [
self.ALL_PROJECTS[prj]['state_name'] for prj in self.projects
]
for prg in self.programmes:
states += list(self.ALL_PROGRAMMES[prg]['states'])
return list(set(states))
def prepare_programme_status(self, obj):
statuses = [
self.ALL_PROGRAMMES[programme]['status']
for programme in self.programmes
]
# Add programme status from projects also
statuses += [
self.ALL_PROGRAMMES[self.ALL_PROJECTS[prj_code]['prg_code']]
['status'] for prj_code in self.projects
]
return list(set(statuses))
def prepare_project_status(self, obj):
return list(
set([
self.ALL_PROJECTS[project_code]['prj_status']
for project_code in self.projects
]))
def prepare_programme_area_ss(self, obj):
areas = [self.ALL_PROJECTS[prj]['area'] for prj in self.projects]
for prg in self.programmes:
areas += list(self.ALL_PROGRAMMES[prg]['areas'])
return list(set(areas))
def prepare_priority_sector_ss(self, obj):
sectors = [self.ALL_PROJECTS[prj]['sector'] for prj in self.projects]
for prg in self.programmes:
sectors += list(self.ALL_PROGRAMMES[prg]['sectors'])
return list(set(sectors))
def prepare_programme_name(self, obj):
prg_codes = list(self.programmes)
prg_codes += [
self.ALL_PROJECTS[project_code]['prg_code']
for project_code in self.projects
]
return [
'{}: {}'.format(
prg_code,
' '.join(self.ALL_PROGRAMMES[prg_code]['name'].split()))
for prg_code in set(prg_codes)
]
def prepare_project_name(self, obj):
return [
'{}: {}'.format(
project_code,
' '.join(self.ALL_PROJECTS[project_code]['prj_name'].split()))
for project_code in self.projects
]
def prepare_role_ss(self, obj):
return [self.ALL_ROLES[role_code] for role_code in self.roles]
def prepare_geotarget(self, obj):
# obj.nuts and obj.geotarget can be empty string
if not obj.nuts:
if obj.geotarget:
return [obj.geotarget]
else:
return None
if len(obj.nuts) > 2:
return [
'{}: {}, {}'.format(obj.nuts, obj.geotarget,
STATES[obj.nuts[:2]])
]
else:
return ['{}: {}'.format(obj.nuts, obj.geotarget)]
def prepare_org_name(self, obj):
return ' '.join(obj.name.split())
def prepare(self, obj):
self.projects = list()
self.programmes = list()
self.roles = set()
for role in obj.roles.all():
if role.project_id and self.ALL_PROJECTS[role.project_id].get(
'prj_code'):
# check prj_code because self.ALL_PROJECTS is defaultdict
self.projects.append(role.project_id)
elif role.programme_id and self.ALL_PROGRAMMES[
role.programme_id].get('name'):
# check programme name because self.ALL_PROGRAMMES is defaultdict
self.programmes.append(role.programme_id)
if role.organisation_role_id != 'DS' and (role.project_id
or role.programme_id):
# Skip donor states and organisations with no project nor programme
self.roles.add(role.organisation_role_id)
if len(self.roles) == 0:
raise exceptions.SkipDocument
self.prepared_data = super().prepare(obj)
# Add extra data in text field to avoid extra queries in template
self.prepared_data['text'] += ' '.join(
self.prepared_data['state_name'] +
self.prepared_data['programme_name'] +
self.prepared_data['project_name'])
self.prepared_data['programme_name_auto'] = (
' '.join(self.prepared_data['programme_name'])
if self.prepared_data['programme_name'] else None)
self.prepared_data['project_name_auto'] = (
' '.join(self.prepared_data['project_name'])
if self.prepared_data['project_name'] else None)
self.prepared_data['geotarget_auto'] = (' '.join(
self.prepared_data['geotarget']) if self.prepared_data['geotarget']
else None)
self.prepared_data['org_name_auto'] = self.prepared_data['org_name']
if self.prepared_data['role_ss']:
self.prepared_data['role_max_priority_code'] = reduce(
lambda max_value, role: max(max_value,
self.ROLE_PRIORITIES.get(role, 0)),
self.prepared_data['role_ss'], 0)
else:
self.prepared_data['role_max_priority_code'] = None
return self.prepared_data
class ProjectIndex(indexes.SearchIndex, indexes.Indexable):
# common facets
state_name = indexes.FacetMultiValueField(model_attr='state__name')
financial_mechanism_ss = indexes.FacetMultiValueField()
programme_area_ss = indexes.FacetMultiValueField()
priority_sector_ss = indexes.FacetMultiValueField()
programme_name = indexes.FacetMultiValueField()
programme_status = indexes.FacetMultiValueField(
model_attr='programme__status')
outcome_ss = indexes.FacetMultiValueField()
outcome_ss_auto = indexes.EdgeNgramField()
kind = indexes.FacetCharField()
# specific facets
code = indexes.FacetCharField(model_attr='code')
project_status = indexes.FacetMultiValueField(model_attr='status')
geotarget = indexes.FacetCharField(model_attr='geotarget')
geotarget_auto = indexes.EdgeNgramField(model_attr='geotarget')
theme_ss = indexes.FacetMultiValueField()
# specific fields
text = indexes.CharField(document=True, use_template=True)
summary = indexes.CharField(model_attr='summary', indexed=False)
url = indexes.CharField(model_attr='url', indexed=False, null=True)
name = indexes.CharField(model_attr='name', indexed=False)
grant = indexes.DecimalField(model_attr='allocation')
def index_queryset(self, using=None):
return (
self.get_model().objects.select_related(
'financial_mechanism',
'outcome',
'programme',
'programme_area',
'programme_area__priority_sector',
'state',
).prefetch_related('themes')
# using prefetch_related may require --batch-size 999 to avoid
# sqlite3.OperationalError: too many SQL variables
)
def get_model(self):
return Project
def prepare_kind(self, obj):
return 'Project'
def prepare_financial_mechanism_ss(self, obj):
return [obj.financial_mechanism.grant_name]
def prepare_programme_area_ss(self, obj):
return [obj.programme_area.name]
def prepare_priority_sector_ss(self, obj):
return [obj.programme_area.priority_sector.name]
def prepare_programme_name(self, obj):
return [
'{}: {}'.format(obj.programme.code,
' '.join(obj.programme.name.split()))
]
def prepare_outcome_ss(self, obj):
return [' '.join(obj.outcome.name.split())]
def prepare_geotarget(self, obj):
if len(obj.nuts) > 2:
return [
'{}: {}, {}'.format(obj.nuts, obj.geotarget,
STATES[obj.nuts[:2]])
]
else:
return ['{}: {}'.format(obj.nuts, obj.geotarget)]
def prepare_theme_ss(self, obj):
return list(set([theme.name for theme in obj.themes.all()]))
def prepare(self, obj):
self.prepared_data = super().prepare(obj)
self.prepared_data['outcome_ss_auto'] = (
' '.join(self.prepared_data['outcome_ss'])
if self.prepared_data['outcome_ss'] else None)
self.prepared_data['geotarget_auto'] = (' '.join(
self.prepared_data['geotarget']) if self.prepared_data['geotarget']
else None)
return self.prepared_data