class CompanyDocument(DocType):
id = fields.IntegerField(attr='id')
name = fields.TextField(
analyzer=edgengram_analyzer,
fields={
'raw': fields.KeywordField(),
}
)
geo = fields.GeoPointField(attr='geo_es_format')
description = fields.TextField()
avatar = fields.FileField()
city = fields.ObjectField(
properties={
'id': fields.IntegerField(),
'name': fields.TextField(
fields={
'raw': fields.KeywordField(),
}
),
'name_fa': fields.TextField(
fields={
'raw': fields.KeywordField(),
}
),
'geo': fields.GeoPointField(
attr='geo_es_format'
),
'region': fields.TextField(
attr='region.name',
fields={
'raw': fields.KeywordField(),
}
)
}
)
related_models = [City, ]
class Meta:
model = Company
fields = [
'slug',
'description_extra',
'size',
'website',
'founded',
'facebook',
'twitter',
'linkedin',
'verified',
]
class PointDocument(DocType):
# address = fields.TextField(
# analyzer=address_analyzer,
# fields={'raw': fields.KeywordField()}
# )
location = fields.GeoPointField()
def prepare_location(self, instance):
return {"lat": instance.latitude, "lon": instance.longitude}
class Meta:
model = Point # The model associated with this DocType
# The fields of the model you want to be indexed in Elasticsearch
fields = [
'store_type',
'identity',
'province',
'city',
'district',
'name',
'address',
'linkman',
'telephone',
'service_time',
]
ignore_signals = False
auto_refresh = True
class CityDocument(Document):
class Index:
name = 'city'
geocoords = fields.GeoPointField()
def prepare_geocoords(self, instance):
return {
'lon': instance.longitude,
'lat': instance.latitude
}
class Django:
model = City
fields = [
'country',
'postal_code',
'place_name',
'admin_name1',
'admin_code1',
'admin_name2',
'admin_code2',
'admin_name3',
'admin_code3',
'latitude',
'longitude',
'accuracy'
]
class PublisherDocument(Document):
"""Publisher Elasticsearch document."""
id = fields.IntegerField(attr='id')
name = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
info = StringField()
address = StringField(fields={'raw': KeywordField()})
city = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
state_province = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
country = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
website = StringField()
# Location
location = fields.GeoPointField(attr='location_field_indexing')
# Geo-shape fields
location_point = fields.GeoShapeField(strategy='recursive',
attr='location_point_indexing')
location_circle = fields.GeoShapeField(strategy='recursive',
attr='location_circle_indexing')
class Django(object):
model = Publisher # The model associate with this Document
class Meta(object):
parallel_indexing = True
class JobDocument(DocType):
# ID
id = fields.IntegerField(attr='id')
title = StringField(analyzer=html_strip, fields={
'raw': KeywordField(),
})
description = fields.TextField()
have_daily_perks = fields.BooleanField()
daily_perks_budget = fields.DoubleField()
have_transportation = fields.BooleanField()
transportation_budget = fields.DoubleField()
have_meal = fields.BooleanField()
meal_budget = fields.DoubleField()
have_space_rest = fields.BooleanField()
space_rest_budget = fields.DoubleField()
is_male = fields.BooleanField()
is_female = fields.BooleanField()
age = fields.IntegerField()
hide_company = fields.BooleanField()
latitude = fields.GeoPointField()
longitude = fields.GeoPointField()
slug = fields.StringField()
publish_date = fields.DateField()
tags = fields.ObjectField(attr='tag_field_indexing',
properties={
'id': fields.IntegerField(),
'name': fields.StringField()
})
company = fields.ObjectField(attr='company_indexing',
properties={
'name': fields.StringField(),
'avatar': fields.StringField(),
'slug': fields.StringField(),
'pk': fields.IntegerField()
})
class Django:
model = Job
class PublisherDocument(DocType):
"""Publisher Elasticsearch document."""
id = fields.IntegerField(attr='id')
name = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
info = StringField()
address = StringField(fields={'raw': KeywordField()})
city = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
state_province = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
country = StringField(
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion),
})
website = StringField()
# Location
location = fields.GeoPointField(attr='location_field_indexing')
class Meta(object):
"""Meta options."""
model = Publisher # The model associate with this DocType
parallel_indexing = True
def regions_field(**kwargs):
return fields.NestedField(properties={
'region_id':
fields.IntegerField(),
'name':
TranslatedTextField('name'),
'hierarchy_label':
TranslatedTextField('hierarchy_label'),
'bbox':
fields.GeoShapeField('wkt_bbox'),
'coords':
fields.GeoPointField()
},
**kwargs)
class RestaurantDocument(Document):
id = fields.IntegerField(attr='id')
name = fields.TextField(analyzer=autocomplete, attr='name')
yelp_image = fields.TextField(attr="yelp_image")
yelp_url = fields.TextField(attr="yelp_url")
url = fields.TextField(attr="url")
rating = fields.FloatField(attr="rating")
price = fields.IntegerField(attr="price")
address = fields.TextField(attr="location")
location = fields.GeoPointField(attr="location_indexing")
address1 = fields.TextField(attr='address1')
phone = fields.TextField(attr='phone')
city = fields.TextField(attr='city')
state = fields.TextField(attr='state')
categories = fields.NestedField(
attr="categories_indexing",
properties={
'id': fields.IntegerField(),
'label': fields.TextField(analyzer=autocomplete),
'api_label': fields.TextField(analyzer=autocomplete)
},
multi=True)
review_count = fields.IntegerField(attr='review_count')
option_count = fields.IntegerField(attr='option_count')
comment_count = fields.IntegerField(attr='comment_count')
open_hours = fields.NestedField(attr='hours_indexing',
properties={
'open': fields.IntegerField(),
'close': fields.IntegerField()
},
multi=True)
class Index:
name = 'restaurants'
settings = {"number_of_shards": 1, "number_of_replicas": 1}
class Django:
model = Restaurant
related_models = [RestaurantCategory]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, RestaurantCategory):
return related_instance.restaurant
class ArticleDocument(DocType):
"""Article elasticsearch document"""
id = fields.StringField(attr='id')
codeInsee = fields.StringField(attr='codeInsee')
pivotLocal = fields.TextField(analyzer=html_strip,
fields={
'raw':
fields.TextField(analyzer='keyword'),
})
nom = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
editeurSource = fields.TextField(analyzer=html_strip,
fields={
'raw':
fields.TextField(analyzer='keyword'),
})
dateMiseAjour = fields.DateField()
nomCommune = fields.TextField(analyzer=html_strip,
fields={
'raw':
fields.TextField(analyzer='keyword'),
})
codePostal = fields.StringField(analyzer=html_strip,
fields={
'raw':
fields.StringField(analyzer='keyword'),
})
telephone = fields.StringField(analyzer=html_strip,
fields={
'raw':
fields.StringField(analyzer='keyword'),
})
email = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
location = fields.GeoPointField(attr='location_field_indexing')
class Meta:
model = articles_models.Article
def get_place_field(options: Optional[Dict] = {}) -> fields.ObjectField:
return fields.ObjectField(
properties={
"id":
fields.IntegerField(),
"address":
fields.TextField(fields={"raw": fields.KeywordField()}, **options),
"geo":
fields.GeoPointField(),
"country":
fields.ObjectField(
properties={
"name":
fields.TextField(fields={
"raw": fields.KeywordField(),
},
**options)
}),
})
class AddressDocument(DocType):
"""Address Elasticsearch document."""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
street = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
house_number = StringField(analyzer=html_strip)
appendix = StringField(analyzer=html_strip)
zip_code = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
# ********************************************************************
# ********** Additional fields for search and filtering **************
# ********************************************************************
# City object
city = fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing'),
'country':
fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing')
})
})
# Country object
country = fields.NestedField(
attr='country_indexing',
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'city':
fields.ObjectField(properties={
'name':
StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
},
),
}, ),
},
)
# Continent object
continent = fields.NestedField(
attr='continent_indexing',
properties={
'id':
fields.IntegerField(),
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'country':
fields.NestedField(
properties={
'id':
fields.IntegerField(),
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
}),
'city':
fields.NestedField(
properties={
'id':
fields.IntegerField(),
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
})
})
})
})
location = fields.GeoPointField(attr='location_field_indexing')
class Meta(object):
"""Meta options."""
model = Address # The model associate with this DocType
class LocationDocument(DocType):
"""
Location document.
"""
# Full fields
__full_fields = {
"raw": KeywordField(),
# edge_ngram_completion
"q": StringField(
analyzer=edge_ngram_completion
),
}
if ELASTICSEARCH_GTE_5_0:
__full_fields.update(
{
"suggest": fields.CompletionField(),
"context": fields.CompletionField(
contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]
),
}
)
full = StringField(
analyzer=html_strip,
fields=__full_fields
)
# Partial fields
__partial_fields = {
"raw": KeywordField(),
# edge_ngram_completion
"q": StringField(
analyzer=edge_ngram_completion
),
}
if ELASTICSEARCH_GTE_5_0:
__partial_fields.update(
{
"suggest": fields.CompletionField(),
"context": fields.CompletionField(
contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]
),
}
)
partial = StringField(
analyzer=html_strip,
fields=__partial_fields
)
# Postcode
__postcode_fields = {
"raw": KeywordField(),
}
if ELASTICSEARCH_GTE_5_0:
__postcode_fields.update(
{
"suggest": fields.CompletionField(),
"context": fields.CompletionField(
contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]
),
}
)
postcode = StringField(
analyzer=html_strip,
fields=__postcode_fields
)
# Number
number = StringField(
attr="address_no",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# Address
address = StringField(
attr="address_street",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# Town
town = StringField(
attr="address_town",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# Authority
authority = StringField(
attr="authority_name",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# URL fields /geocode/slug
geocode = StringField(
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# Slug
slug = StringField(
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# ********************* Filter fields **********************
# Category
category = StringField(
attr="group",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# Occupied
occupied = StringField(
attr="occupation_status_text",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
size = fields.FloatField(attr="floor_area")
staff = fields.FloatField(attr="employee_count")
rent = fields.FloatField(attr="rental_valuation")
revenue = fields.FloatField(attr="revenue")
coordinates = fields.GeoPointField(attr="location_field_indexing")
class Meta(object):
"""Meta options."""
model = Location # The model associate with this DocType
parallel_indexing = True
queryset_pagination = 50 # This will split the queryset
class CityDocument(Document):
"""City Elasticsearch document.
This document has been created purely for testing out complex fields.
"""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# ********************** Main data fields for search *****************
# ********************************************************************
name = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
info = StringField(analyzer=html_strip)
# ********************************************************************
# ************** Nested fields for search and filtering **************
# ********************************************************************
# City object
country = fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing'),
})
location = fields.GeoPointField(attr='location_field_indexing')
# ********************************************************************
# ********** Other complex fields for search and filtering ***********
# ********************************************************************
boolean_list = fields.ListField(StringField(attr='boolean_list_indexing'))
# boolean_dict_indexing = fields.ObjectField(
# properties={
# 'true': fields.BooleanField(),
# 'false': fields.BooleanField(),
# }
# )
datetime_list = fields.ListField(
StringField(attr='datetime_list_indexing'))
# datetime_dict_indexing
float_list = fields.ListField(StringField(attr='float_list_indexing'))
# float_dict_indexing
integer_list = fields.ListField(StringField(attr='integer_list_indexing'))
# integer_dict_indexing
class Django(object):
model = City # The model associate with this Document
class Meta(object):
parallel_indexing = True
class ActivityDocument(DocType):
title_keyword = fields.KeywordField(attr='title')
title = fields.TextField(fielddata=True)
description = fields.TextField()
status = fields.KeywordField()
status_score = fields.FloatField()
created = fields.DateField()
date = fields.DateField()
deadline = fields.DateField()
type = fields.KeywordField()
owner = fields.NestedField(properties={
'id': fields.KeywordField(),
'full_name': fields.TextField()
})
initiative = fields.NestedField(properties={
'title': fields.TextField(),
'pitch': fields.TextField(),
'story': fields.TextField(),
})
theme = fields.NestedField(
attr='initiative.theme',
properties={
'id': fields.KeywordField(),
}
)
categories = fields.NestedField(
attr='initiative.categories',
properties={
'id': fields.LongField(),
'slug': fields.KeywordField(),
}
)
position = fields.GeoPointField()
country = fields.KeywordField()
expertise = fields.NestedField(
properties={
'id': fields.KeywordField(),
}
)
segments = fields.NestedField(
properties={
'id': fields.KeywordField(),
'type': fields.KeywordField(attr='type.slug'),
'name': fields.TextField()
}
)
location = fields.NestedField(
attr='location',
properties={
'id': fields.LongField(),
'formatted_address': fields.TextField(),
}
)
initiative_location = fields.NestedField(
attr='initiative.location',
properties={
'id': fields.LongField(),
'name': fields.TextField(),
'city': fields.TextField(),
}
)
contributions = fields.DateField()
contribution_count = fields.IntegerField()
activity_date = fields.DateField()
class Meta(object):
model = Activity
def get_queryset(self):
return super(ActivityDocument, self).get_queryset().select_related(
'initiative', 'owner',
)
@classmethod
def search(cls, using=None, index=None):
# Use search class that supports polymorphic models
return Search(
using=using or cls._doc_type.using,
index=index or cls._doc_type.index,
doc_type=[cls],
model=cls._doc_type.model
)
def prepare_contributions(self, instance):
return [
contribution.created for contribution
in instance.contributions.filter(status__in=('new', 'success'))
]
def prepare_type(self, instance):
return str(instance.__class__.__name__.lower())
def prepare_contribution_count(self, instance):
return len(instance.contributions.filter(status__in=('new', 'success')))
def prepare_country(self, instance):
if instance.initiative.location:
return instance.initiative.location.country_id
if instance.initiative.place:
return instance.initiative.place.country_id
def prepare_location(self, instance):
if hasattr(instance, 'location') and instance.location:
return {
'id': instance.location.pk,
'formatted_address': instance.location.formatted_address
}
def prepare_expertise(self, instance):
if hasattr(instance, 'expertise') and instance.expertise:
return {'id': instance.expertise_id}
def prepare_position(self, instance):
return None
def prepare_deadline(self, instance):
return None
def prepare_date(self, instance):
return None
class CollectionItemDocument(DocType):
"""Collection item document."""
# ID
id = fields.IntegerField(attr='id')
record_number = KeywordField()
inventory_number = KeywordField()
api_url = KeywordField(index="not_analyzed")
web_url = KeywordField(index="not_analyzed")
# ********************************************************************
# *************** Main data fields for search and filtering **********
# ********************************************************************
importer_uid = KeywordField(attr='importer_uid_indexing')
language_code_orig = KeywordField(attr='language_code_orig')
department = StringField(
attr='department_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
# ********************************************************************
# ***************************** English ******************************
# ********************************************************************
title_en = StringField(
attr='title_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
description_en = StringField(
attr='description_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
period_en = StringField(
attr='period_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
period_1_en = fields.NestedField(
attr='period_1_en_indexing',
properties={
'name':
StringField(analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
}),
'period_2_en':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
}),
'period_3_en':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
}),
'period_4_en':
fields.NestedField(
properties={
'name':
StringField(
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
})
})
})
})
})
primary_object_type_en = StringField(
attr='primary_object_type_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
'suggest': fields.CompletionField(),
})
object_type_en = StringField(attr='object_type_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural':
StringField(analyzer='english'),
'suggest': fields.CompletionField(),
})
# To be shown on the detail page
object_type_detail_en = fields.TextField(
attr='object_type_detail_en_indexing', index='no')
material_en = StringField(
attr='material_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
material_detail_en = fields.TextField(attr='material_detail_en_indexing',
index='no')
city_en = StringField(
attr='city_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
country_en = StringField(
attr='country_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
references_en = fields.TextField(attr='references_en_indexing', index='no')
# To be shown on the detail page
acquired_en = fields.TextField(attr='acquired_en_indexing', index='no')
# To be shown on the detail page
site_found_en = fields.TextField(attr='site_found_en_indexing', index='no')
# To be shown on the detail page
reign_en = fields.TextField(attr='reign_en_indexing', index='no')
# To be shown on the detail page
keywords_en = fields.TextField(attr='keywords_en_indexing', index='no')
# To be shown on the detail page
dynasty_en = fields.TextField(attr='dynasty_en_indexing', index='no')
# New fields
# To be shown on the detail page
credit_line_en = fields.TextField(attr='credit_line_en_indexing',
index='no')
# To be shown on the detail page
region_en = fields.TextField(attr='region_en_indexing', index='no')
# To be shown on the detail page
sub_region_en = fields.TextField(attr='sub_region_en_indexing', index='no')
# To be shown on the detail page
locale_en = fields.TextField(attr='locale_en_indexing', index='no')
# To be shown on the detail page
excavation_en = fields.TextField(attr='excavation_en_indexing', index='no')
# To be shown on the detail page
museum_collection_en = fields.TextField(
attr='museum_collection_en_indexing', index='no')
# To be shown on the detail page
style_en = fields.TextField(attr='style_en_indexing', index='no')
# To be shown on the detail page
culture_en = fields.TextField(attr='culture_en_indexing', index='no')
# To be shown on the detail page
inscriptions_en = fields.TextField(attr='inscriptions_en_indexing',
index='no')
# To be shown on the detail page
provenance_en = fields.TextField(attr='provenance_en_indexing', index='no')
# To be shown on the detail page
exhibitions_en = fields.TextField(attr='exhibitions_en_indexing',
index='no')
# ********************************************************************
# ****************************** Dutch *******************************
# ********************************************************************
title_nl = StringField(
attr='title_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
description_nl = StringField(
attr='description_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
period_nl = StringField(
attr='period_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
period_1_nl = fields.NestedField(
attr='period_1_nl_indexing',
properties={
'name':
StringField(analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
}),
'period_2_nl':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
}),
'period_3_nl':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
}),
'period_4_nl':
fields.NestedField(
properties={
'name':
StringField(
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
})
})
})
})
})
primary_object_type_nl = StringField(
attr='primary_object_type_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
'suggest': fields.CompletionField(),
})
object_type_nl = StringField(attr='object_type_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
'suggest': fields.CompletionField(),
})
# To be shown on the detail page
object_type_detail_nl = fields.TextField(
attr='object_type_detail_nl_indexing', index='no')
material_nl = StringField(
attr='material_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
material_detail_nl = fields.TextField(attr='material_detail_nl_indexing',
index='no')
city_nl = StringField(
attr='city_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
country_nl = StringField(
attr='country_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
keywords_nl = fields.TextField(attr='keywords_nl_indexing', index='no')
# To be shown on the detail page
acquired_nl = fields.TextField(attr='acquired_nl_indexing', index='no')
# To be shown on the detail page
site_found_nl = fields.TextField(attr='site_found_nl_indexing', index='no')
# To be shown on the detail page
reign_nl = fields.TextField(attr='reign_nl_indexing', index='no')
# To be shown on the detail page
references_nl = fields.TextField(attr='references_nl_indexing', index='no')
# To be shown on the detail page
dynasty_nl = fields.TextField(attr='dynasty_nl_indexing', index='no')
# New fields
# To be shown on the detail page
credit_line_nl = fields.TextField(attr='credit_line_nl_indexing',
index='no')
# To be shown on the detail page
region_nl = fields.TextField(attr='region_nl_indexing', index='no')
# To be shown on the detail page
sub_region_nl = fields.TextField(attr='sub_region_nl_indexing', index='no')
# To be shown on the detail page
locale_nl = fields.TextField(attr='locale_nl_indexing', index='no')
# To be shown on the detail page
excavation_nl = fields.TextField(attr='excavation_nl_indexing', index='no')
# To be shown on the detail page
museum_collection_nl = fields.TextField(
attr='museum_collection_nl_indexing', index='no')
# To be shown on the detail page
style_nl = fields.TextField(attr='style_nl_indexing', index='no')
# To be shown on the detail page
culture_nl = fields.TextField(attr='culture_nl_indexing', index='no')
# To be shown on the detail page
inscriptions_nl = fields.TextField(attr='inscriptions_nl_indexing',
index='no')
# To be shown on the detail page
provenance_nl = fields.TextField(attr='provenance_nl_indexing', index='no')
# To be shown on the detail page
exhibitions_nl = fields.TextField(attr='exhibitions_nl_indexing',
index='no')
# ********************************************************************
# ************************** Language independent ********************
# ********************************************************************
dimensions = StringField(
attr='dimensions_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(),
# 'suggest': fields.CompletionField(),
})
object_date_begin = StringField(
attr='object_date_begin_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(),
# 'suggest': fields.CompletionField(),
})
object_date_end = StringField(
attr='object_date_end_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(),
# 'suggest': fields.CompletionField(),
})
location = fields.GeoPointField(attr='geo_location_indexing')
# List of 32x32 PNG versions of the images. Full path to.
images = fields.ListField(StringField(attr='images_indexing'))
# List of image URLs.
images_urls = fields.ListField(
fields.ObjectField(attr='images_urls_indexing',
properties={
'th': KeywordField(index="not_analyzed"),
'lr': KeywordField(index="not_analyzed"),
}))
# Classified as by our AI
classified_as = fields.ListField(
StringField(attr='classified_as_indexing',
fields={
'raw': KeywordField(),
}))
# Classified as 1st element
classified_as_1 = StringField(attr='classified_as_1_indexing',
fields={
'raw': KeywordField(),
})
# Classified as 2nd element
classified_as_2 = StringField(attr='classified_as_2_indexing',
fields={
'raw': KeywordField(),
})
# Classified as 3rd element
classified_as_3 = StringField(attr='classified_as_3_indexing',
fields={
'raw': KeywordField(),
})
# ********************************************************************
# ************** Nested fields for search and filtering **************
# ********************************************************************
# # City object
# country = fields.NestedField(
# properties={
# 'name': StringField(
# analyzer=html_strip,
# fields={
# 'raw': KeywordField(),
# 'suggest': fields.CompletionField(),
# }
# ),
# 'info': StringField(analyzer=html_strip),
# 'location': fields.GeoPointField(attr='location_field_indexing'),
# }
# )
#
# location = fields.GeoPointField(attr='location_field_indexing')
class Meta(object):
"""Meta options."""
model = Item # The model associate with this DocType
def get_queryset(self):
"""Filter out items that are not eligible for indexing."""
qs = super(CollectionItemDocument, self).get_queryset()
# qs = qs.select_related('period_node').prefetch_related('images')
filters = []
for field in ['title']:
for language in ['en', 'nl']:
filters.extend([
Q(**{"{}_{}__isnull".format(field, language): True}),
Q(**{"{}_{}__exact".format(field, language): ''}),
])
if filters:
qs = qs.exclude(six.moves.reduce(operator.or_, filters))
# We concatenate ``object_type`` and ``classification`` fields, after
# cleaning them. Therefore, db-only checks don't work here.
ids = []
for item in qs:
if not (item.object_type_en_indexing
and item.object_type_nl_indexing):
ids.append(item.pk)
return qs.exclude(id__in=ids)
def prepare_department(self, instance):
"""Prepare department."""
return instance.department_indexing \
if instance.department_indexing\
else VALUE_NOT_SPECIFIED
def prepare_object_date_begin(self, instance):
"""Prepare material."""
return instance.object_date_begin_indexing
def prepare_object_date_end(self, instance):
"""Prepare material."""
return instance.object_date_end_indexing
# ********************************************************************
# ***************************** English ******************************
# ********************************************************************
def prepare_material_en(self, instance):
"""Prepare material."""
return instance.material_en_indexing \
if instance.material_en_indexing\
else VALUE_NOT_SPECIFIED
def prepare_period_en(self, instance):
"""Prepare state."""
return instance.period_en_indexing \
if instance.period_en_indexing \
else VALUE_NOT_SPECIFIED
def prepare_dynasty_en(self, instance):
"""Prepare dynasty."""
return instance.dynasty_en_indexing \
if instance.dynasty_en_indexing \
else VALUE_NOT_SPECIFIED
def prepare_description_en(self, instance):
"""Prepare description."""
return instance.description_en_indexing \
if instance.description_en_indexing\
else VALUE_NOT_SPECIFIED
def prepare_city_en(self, instance):
"""Prepare city."""
return instance.city_en_indexing \
if instance.city_en_indexing\
else VALUE_NOT_SPECIFIED
def prepare_country_en(self, instance):
"""Prepare country."""
return instance.country_en_indexing \
if instance.country_en_indexing \
else VALUE_NOT_SPECIFIED
# ********************************************************************
# ****************************** Dutch *******************************
# ********************************************************************
def prepare_material_nl(self, instance):
"""Prepare material."""
return instance.material_nl_indexing \
if instance.material_nl_indexing\
else VALUE_NOT_SPECIFIED
def prepare_period_nl(self, instance):
"""Prepare state."""
return instance.period_nl_indexing \
if instance.period_nl_indexing \
else VALUE_NOT_SPECIFIED
def prepare_dynasty_nl(self, instance):
"""Prepare dynasty."""
return instance.dynasty_nl_indexing \
if instance.dynasty_nl_indexing \
else VALUE_NOT_SPECIFIED
def prepare_description_nl(self, instance):
"""Prepare description."""
return instance.description_nl_indexing \
if instance.description_nl_indexing\
else VALUE_NOT_SPECIFIED
def prepare_city_nl(self, instance):
"""Prepare city."""
return instance.city_nl_indexing \
if instance.city_nl_indexing\
else VALUE_NOT_SPECIFIED
def prepare_country_nl(self, instance):
"""Prepare country."""
return instance.country_nl_indexing \
if instance.country_nl_indexing \
else VALUE_NOT_SPECIFIED
class LocationDocument(DocType):
"""
Location document.
"""
full = fields.StringField(
analyzer=html_strip,
fields={
"raw": KeywordField(),
"suggest": fields.CompletionField(),
"context": fields.CompletionField(
contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]
),
# edge_ngram_completion
"q": StringField(
analyzer=edge_ngram_completion
),
}
)
partial = fields.StringField(
analyzer=html_strip,
fields={
"raw": KeywordField(),
"suggest": fields.CompletionField(),
"context": fields.CompletionField(
contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]
),
# edge_ngram_completion
"q": StringField(
analyzer=edge_ngram_completion
),
}
)
postcode = fields.StringField(
analyzer=html_strip,
fields={
"raw": KeywordField(),
"suggest": fields.CompletionField(),
"context": fields.CompletionField(
contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]
),
}
)
number = fields.StringField(
attr="address_no",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
address = fields.StringField(
attr="address_street",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
town = fields.StringField(
attr="address_town",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
authority = fields.StringField(
attr="authority_name",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# URL fields /geocode/slug
geocode = fields.StringField(
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
slug = fields.StringField(
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
# Filter fields
category = fields.StringField(
attr="group",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
occupied = fields.StringField(
attr="occupation_status_text",
analyzer=html_strip,
fields={
"raw": KeywordField(),
}
)
size = fields.FloatField(attr="floor_area")
staff = fields.FloatField(attr="employee_count")
rent = fields.FloatField(attr="rental_valuation")
revenue = fields.FloatField(attr="revenue")
coordinates = fields.GeoPointField(attr="location_field_indexing")
class Meta(object):
"""Meta options."""
model = Location # The model associate with this DocType
parallel_indexing = True
queryset_pagination = 50 # This will split the queryset
class DeviceDocument(DocType):
date_modified = fields.DateField()
user = fields.ObjectField(properties={
'mobile': fields.StringField(),
'email': fields.StringField(),
})
location = fields.GeoPointField()
tag_suggest = fields.CompletionField()
stores = fields.NestedField(properties={
'store_id': fields.StringField(),
'date': fields.DateField(),
})
'''offers = fields.NestedField(properties={
'reward': fields.StringField(),
'date_redeemed': fields.DateField(),
'expiry_date': fields.DateField(),
'date_claimed': fields.DateField(),
})'''
class Meta:
model = Device # The model associated with this DocType
# The fields of the model you want to be indexed in Elasticsearch
fields = [
'device_id',
'date_created',
'active',
]
related_models = [User]
# Ignore auto updating of Elasticsearch when a model is saved
# or deleted:
# ignore_signals = True
# Don't perform an index refresh after every update (overrides global setting):
# auto_refresh = False
# Paginate the django queryset used to populate the index with the specified size
# (by default there is no pagination)
# queryset_pagination = 5000
def get_instances_from_related(self, related_instance):
"""If related_models is set, define how to retrieve the Car instances from the related model."""
return related_instance.device_set.all()
def update_stores(self, store_id):
# https://github.com/istresearch/kibana-object-format
self.stores.append({'store_id': store_id, 'date': datetime.now()})
self.save()
def prepare_stores(self, instance):
#print instance.stores
try:
device = self.get(id=instance.pk)
return device.stores if device.stores else []
except TransportError:
return []
def prepare_offers(self, instance):
return []
def prepare_tag_suggest(self, instance):
return [
"red",
"tiny",
]
def prepare_location(self, instance):
return dict(lat=10.0089331, lon=76.3155752)
def prepare_date_modified(self, instance):
return instance.date_modified
class PublicSource(PublicSourceBase):
id = fields.KeywordField(attr='id')
name = fields.TextField(
attr='name',
fields={'keyword': fields.KeywordField(
normalizer=basic_normalizer
)},
index_prefixes={}
)
type = fields.IntegerField(attr='type.id')
is_private = fields.BooleanField(attr='is_private')
is_public = fields.BooleanField()
parent = fields.TextField(
attr='parent.name',
fields={'keyword': fields.KeywordField(
normalizer=basic_normalizer
)},
index_prefixes={}
)
attributes = fields.ObjectField()
collections = fields.ObjectField()
credits = fields.ObjectField()
geo_location = fields.GeoPointField()
class Index:
name = 'public_sources'
class Django:
model = Source
def get_queryset(self):
return Source.objects.filter(type=13, workflow__is_public=True)
def prepare_is_public(self, instance):
try:
return instance.workflow.is_public
except ObjectDoesNotExist:
return False
def prepare_attributes(self, instance):
attribute_list = []
dates_list = []
for attribute in instance.attributes.all():
label = attribute.attribute_type.short_name
if attribute.attribute_type.data_type == 'DATE':
if attribute.value_DATE:
dates_list.append(attribute.value_DATE.strftime('%Y-%m-%d'))
elif attribute.value_DATE_y:
d = attribute.value_DATE_d or 1
m = attribute.value_DATE_m or 1
y = attribute.value_DATE_y
dates_list.append(date(y, m, d).strftime('%Y-%m-%d'))
elif attribute.attribute_type.data_type == 'TXT':
attribute_list.append((label, attribute.value_TXT))
else:
attribute_list.append((label, str(attribute)))
if dates_list:
attribute_list.append(('date', dates_list))
if attribute_list:
return dict(attribute_list)
def prepare_collections(self, instance):
try:
if instance.sets.all().count() > 0:
cols = [('name', set.set_id.name) for set in instance.sets.all() if set.set_id.set_type == 2]
if cols:
return dict(cols)
else:
return [{'name': 'none'}]
except ObjectDoesNotExist:
return [{'name': 'none'}]
def prepare_credits(self, instance):
credit_list = []
for credit in instance.credits.all():
agent = credit.agent.standard_name
type = credit.get_type_display()
credit_list.append({
'agent': f'{agent} ({type})',
'type': type
})
return credit_list
def prepare_geo_location(self, instance):
locales = instance.attributes.filter(attribute_type=36)
if locales.exists():
loc_id = locales[0].value_JSON['id']
locale = LocaleReference.objects.get(id=loc_id)
return f'{locale.latitude},{locale.longitude}'
class AddressDocument(Document):
"""Address Elasticsearch document."""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
__street_fields = {
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
if ELASTICSEARCH_GTE_5_0:
__street_fields.update(
{
'suggest_context': fields.CompletionField(
contexts=[
{
"name": "loc",
"type": "geo",
"path": "location",
"precision": "1000km",
},
]
),
}
)
street = StringField(
analyzer=html_strip,
fields=__street_fields
)
house_number = StringField(analyzer=html_strip)
appendix = StringField(analyzer=html_strip)
zip_code = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
)
# ********************************************************************
# ********** Additional fields for search and filtering **************
# ********************************************************************
# City object
city = fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'info': StringField(analyzer=html_strip),
'location': fields.GeoPointField(attr='location_field_indexing'),
'country': fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'info': StringField(analyzer=html_strip),
'location': fields.GeoPointField(
attr='location_field_indexing'
)
}
)
}
)
# Country object
country = fields.NestedField(
attr='country_indexing',
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'city': fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
},
),
},
),
},
)
# Continent object
continent = fields.NestedField(
attr='continent_indexing',
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'country': fields.NestedField(
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
),
'city': fields.NestedField(
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
}
)
}
)
}
)
location = fields.GeoPointField(
attr='location_field_indexing',
)
class Django(object):
model = Address # The model associate with this Document
class Meta(object):
parallel_indexing = True
class AddressDocument(Document):
id = fields.IntegerField(attr='id')
street = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
house_number = StringField(analyzer=html_strip)
appendix = StringField(analyzer=html_strip)
zip_code = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
city = fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField()
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing'),
'country':
fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing')
})
})
# Defining the ``@property`` functions in the address model
country = fields.NestedField(
attr='country_indexing',
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'city':
fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={'raw': KeywordField()})
})
})
continent = fields.NestedField(
attr='continent_indexing',
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField()
}),
'country':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip,
fields={'raw': KeywordField()}),
'city':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip,
fields={'raw': KeywordField()})
})
})
})
location = fields.GeoPointField(attr='location_field_indexing')
class Django(object):
model = Address
