class CompanyDocument(DocType): id = fields.IntegerField(attr='id') name = fields.TextField( analyzer=edgengram_analyzer, fields={ 'raw': fields.KeywordField(), } ) geo = fields.GeoPointField(attr='geo_es_format') description = fields.TextField() avatar = fields.FileField() city = fields.ObjectField( properties={ 'id': fields.IntegerField(), 'name': fields.TextField( fields={ 'raw': fields.KeywordField(), } ), 'name_fa': fields.TextField( fields={ 'raw': fields.KeywordField(), } ), 'geo': fields.GeoPointField( attr='geo_es_format' ), 'region': fields.TextField( attr='region.name', fields={ 'raw': fields.KeywordField(), } ) } ) related_models = [City, ] class Meta: model = Company fields = [ 'slug', 'description_extra', 'size', 'website', 'founded', 'facebook', 'twitter', 'linkedin', 'verified', ]
class PointDocument(DocType): # address = fields.TextField( # analyzer=address_analyzer, # fields={'raw': fields.KeywordField()} # ) location = fields.GeoPointField() def prepare_location(self, instance): return {"lat": instance.latitude, "lon": instance.longitude} class Meta: model = Point # The model associated with this DocType # The fields of the model you want to be indexed in Elasticsearch fields = [ 'store_type', 'identity', 'province', 'city', 'district', 'name', 'address', 'linkman', 'telephone', 'service_time', ] ignore_signals = False auto_refresh = True
class CityDocument(Document): class Index: name = 'city' geocoords = fields.GeoPointField() def prepare_geocoords(self, instance): return { 'lon': instance.longitude, 'lat': instance.latitude } class Django: model = City fields = [ 'country', 'postal_code', 'place_name', 'admin_name1', 'admin_code1', 'admin_name2', 'admin_code2', 'admin_name3', 'admin_code3', 'latitude', 'longitude', 'accuracy' ]
class PublisherDocument(Document): """Publisher Elasticsearch document.""" id = fields.IntegerField(attr='id') name = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) info = StringField() address = StringField(fields={'raw': KeywordField()}) city = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) state_province = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) country = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) website = StringField() # Location location = fields.GeoPointField(attr='location_field_indexing') # Geo-shape fields location_point = fields.GeoShapeField(strategy='recursive', attr='location_point_indexing') location_circle = fields.GeoShapeField(strategy='recursive', attr='location_circle_indexing') class Django(object): model = Publisher # The model associate with this Document class Meta(object): parallel_indexing = True
class JobDocument(DocType): # ID id = fields.IntegerField(attr='id') title = StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), }) description = fields.TextField() have_daily_perks = fields.BooleanField() daily_perks_budget = fields.DoubleField() have_transportation = fields.BooleanField() transportation_budget = fields.DoubleField() have_meal = fields.BooleanField() meal_budget = fields.DoubleField() have_space_rest = fields.BooleanField() space_rest_budget = fields.DoubleField() is_male = fields.BooleanField() is_female = fields.BooleanField() age = fields.IntegerField() hide_company = fields.BooleanField() latitude = fields.GeoPointField() longitude = fields.GeoPointField() slug = fields.StringField() publish_date = fields.DateField() tags = fields.ObjectField(attr='tag_field_indexing', properties={ 'id': fields.IntegerField(), 'name': fields.StringField() }) company = fields.ObjectField(attr='company_indexing', properties={ 'name': fields.StringField(), 'avatar': fields.StringField(), 'slug': fields.StringField(), 'pk': fields.IntegerField() }) class Django: model = Job
class PublisherDocument(DocType): """Publisher Elasticsearch document.""" id = fields.IntegerField(attr='id') name = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) info = StringField() address = StringField(fields={'raw': KeywordField()}) city = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) state_province = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) country = StringField( fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), 'edge_ngram_completion': StringField( analyzer=edge_ngram_completion), }) website = StringField() # Location location = fields.GeoPointField(attr='location_field_indexing') class Meta(object): """Meta options.""" model = Publisher # The model associate with this DocType parallel_indexing = True
def regions_field(**kwargs): return fields.NestedField(properties={ 'region_id': fields.IntegerField(), 'name': TranslatedTextField('name'), 'hierarchy_label': TranslatedTextField('hierarchy_label'), 'bbox': fields.GeoShapeField('wkt_bbox'), 'coords': fields.GeoPointField() }, **kwargs)
class RestaurantDocument(Document): id = fields.IntegerField(attr='id') name = fields.TextField(analyzer=autocomplete, attr='name') yelp_image = fields.TextField(attr="yelp_image") yelp_url = fields.TextField(attr="yelp_url") url = fields.TextField(attr="url") rating = fields.FloatField(attr="rating") price = fields.IntegerField(attr="price") address = fields.TextField(attr="location") location = fields.GeoPointField(attr="location_indexing") address1 = fields.TextField(attr='address1') phone = fields.TextField(attr='phone') city = fields.TextField(attr='city') state = fields.TextField(attr='state') categories = fields.NestedField( attr="categories_indexing", properties={ 'id': fields.IntegerField(), 'label': fields.TextField(analyzer=autocomplete), 'api_label': fields.TextField(analyzer=autocomplete) }, multi=True) review_count = fields.IntegerField(attr='review_count') option_count = fields.IntegerField(attr='option_count') comment_count = fields.IntegerField(attr='comment_count') open_hours = fields.NestedField(attr='hours_indexing', properties={ 'open': fields.IntegerField(), 'close': fields.IntegerField() }, multi=True) class Index: name = 'restaurants' settings = {"number_of_shards": 1, "number_of_replicas": 1} class Django: model = Restaurant related_models = [RestaurantCategory] def get_instances_from_related(self, related_instance): if isinstance(related_instance, RestaurantCategory): return related_instance.restaurant
class ArticleDocument(DocType): """Article elasticsearch document""" id = fields.StringField(attr='id') codeInsee = fields.StringField(attr='codeInsee') pivotLocal = fields.TextField(analyzer=html_strip, fields={ 'raw': fields.TextField(analyzer='keyword'), }) nom = fields.TextField(analyzer=html_strip, fields={ 'raw': fields.TextField(analyzer='keyword'), }) editeurSource = fields.TextField(analyzer=html_strip, fields={ 'raw': fields.TextField(analyzer='keyword'), }) dateMiseAjour = fields.DateField() nomCommune = fields.TextField(analyzer=html_strip, fields={ 'raw': fields.TextField(analyzer='keyword'), }) codePostal = fields.StringField(analyzer=html_strip, fields={ 'raw': fields.StringField(analyzer='keyword'), }) telephone = fields.StringField(analyzer=html_strip, fields={ 'raw': fields.StringField(analyzer='keyword'), }) email = fields.TextField(analyzer=html_strip, fields={ 'raw': fields.TextField(analyzer='keyword'), }) location = fields.GeoPointField(attr='location_field_indexing') class Meta: model = articles_models.Article
def get_place_field(options: Optional[Dict] = {}) -> fields.ObjectField: return fields.ObjectField( properties={ "id": fields.IntegerField(), "address": fields.TextField(fields={"raw": fields.KeywordField()}, **options), "geo": fields.GeoPointField(), "country": fields.ObjectField( properties={ "name": fields.TextField(fields={ "raw": fields.KeywordField(), }, **options) }), })
class AddressDocument(DocType): """Address Elasticsearch document.""" # In different parts of the code different fields are used. There are # a couple of use cases: (1) more-like-this functionality, where `title`, # `description` and `summary` fields are used, (2) search and filtering # functionality where all of the fields are used. # ID id = fields.IntegerField(attr='id') # ******************************************************************** # *********************** Main data fields for search **************** # ******************************************************************** street = StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }) house_number = StringField(analyzer=html_strip) appendix = StringField(analyzer=html_strip) zip_code = StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }) # ******************************************************************** # ********** Additional fields for search and filtering ************** # ******************************************************************** # City object city = fields.ObjectField( properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }), 'info': StringField(analyzer=html_strip), 'location': fields.GeoPointField(attr='location_field_indexing'), 'country': fields.ObjectField( properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }), 'info': StringField(analyzer=html_strip), 'location': fields.GeoPointField(attr='location_field_indexing') }) }) # Country object country = fields.NestedField( attr='country_indexing', properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }), 'city': fields.ObjectField(properties={ 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), }, ), }, ), }, ) # Continent object continent = fields.NestedField( attr='continent_indexing', properties={ 'id': fields.IntegerField(), 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }), 'country': fields.NestedField( properties={ 'id': fields.IntegerField(), 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), }), 'city': fields.NestedField( properties={ 'id': fields.IntegerField(), 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), }) }) }) }) location = fields.GeoPointField(attr='location_field_indexing') class Meta(object): """Meta options.""" model = Address # The model associate with this DocType
class LocationDocument(DocType): """ Location document. """ # Full fields __full_fields = { "raw": KeywordField(), # edge_ngram_completion "q": StringField( analyzer=edge_ngram_completion ), } if ELASTICSEARCH_GTE_5_0: __full_fields.update( { "suggest": fields.CompletionField(), "context": fields.CompletionField( contexts=[ { "name": "category", "type": "category", "path": "category.raw", }, { "name": "occupied", "type": "category", "path": "occupied.raw", }, ] ), } ) full = StringField( analyzer=html_strip, fields=__full_fields ) # Partial fields __partial_fields = { "raw": KeywordField(), # edge_ngram_completion "q": StringField( analyzer=edge_ngram_completion ), } if ELASTICSEARCH_GTE_5_0: __partial_fields.update( { "suggest": fields.CompletionField(), "context": fields.CompletionField( contexts=[ { "name": "category", "type": "category", "path": "category.raw", }, { "name": "occupied", "type": "category", "path": "occupied.raw", }, ] ), } ) partial = StringField( analyzer=html_strip, fields=__partial_fields ) # Postcode __postcode_fields = { "raw": KeywordField(), } if ELASTICSEARCH_GTE_5_0: __postcode_fields.update( { "suggest": fields.CompletionField(), "context": fields.CompletionField( contexts=[ { "name": "category", "type": "category", "path": "category.raw", }, { "name": "occupied", "type": "category", "path": "occupied.raw", }, ] ), } ) postcode = StringField( analyzer=html_strip, fields=__postcode_fields ) # Number number = StringField( attr="address_no", analyzer=html_strip, fields={ "raw": KeywordField(), } ) # Address address = StringField( attr="address_street", analyzer=html_strip, fields={ "raw": KeywordField(), } ) # Town town = StringField( attr="address_town", analyzer=html_strip, fields={ "raw": KeywordField(), } ) # Authority authority = StringField( attr="authority_name", analyzer=html_strip, fields={ "raw": KeywordField(), } ) # URL fields /geocode/slug geocode = StringField( analyzer=html_strip, fields={ "raw": KeywordField(), } ) # Slug slug = StringField( analyzer=html_strip, fields={ "raw": KeywordField(), } ) # ********************* Filter fields ********************** # Category category = StringField( attr="group", analyzer=html_strip, fields={ "raw": KeywordField(), } ) # Occupied occupied = StringField( attr="occupation_status_text", analyzer=html_strip, fields={ "raw": KeywordField(), } ) size = fields.FloatField(attr="floor_area") staff = fields.FloatField(attr="employee_count") rent = fields.FloatField(attr="rental_valuation") revenue = fields.FloatField(attr="revenue") coordinates = fields.GeoPointField(attr="location_field_indexing") class Meta(object): """Meta options.""" model = Location # The model associate with this DocType parallel_indexing = True queryset_pagination = 50 # This will split the queryset
class CityDocument(Document): """City Elasticsearch document. This document has been created purely for testing out complex fields. """ # In different parts of the code different fields are used. There are # a couple of use cases: (1) more-like-this functionality, where `title`, # `description` and `summary` fields are used, (2) search and filtering # functionality where all of the fields are used. # ID id = fields.IntegerField(attr='id') # ******************************************************************** # ********************** Main data fields for search ***************** # ******************************************************************** name = StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }) info = StringField(analyzer=html_strip) # ******************************************************************** # ************** Nested fields for search and filtering ************** # ******************************************************************** # City object country = fields.NestedField( properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }), 'info': StringField(analyzer=html_strip), 'location': fields.GeoPointField(attr='location_field_indexing'), }) location = fields.GeoPointField(attr='location_field_indexing') # ******************************************************************** # ********** Other complex fields for search and filtering *********** # ******************************************************************** boolean_list = fields.ListField(StringField(attr='boolean_list_indexing')) # boolean_dict_indexing = fields.ObjectField( # properties={ # 'true': fields.BooleanField(), # 'false': fields.BooleanField(), # } # ) datetime_list = fields.ListField( StringField(attr='datetime_list_indexing')) # datetime_dict_indexing float_list = fields.ListField(StringField(attr='float_list_indexing')) # float_dict_indexing integer_list = fields.ListField(StringField(attr='integer_list_indexing')) # integer_dict_indexing class Django(object): model = City # The model associate with this Document class Meta(object): parallel_indexing = True
class ActivityDocument(DocType): title_keyword = fields.KeywordField(attr='title') title = fields.TextField(fielddata=True) description = fields.TextField() status = fields.KeywordField() status_score = fields.FloatField() created = fields.DateField() date = fields.DateField() deadline = fields.DateField() type = fields.KeywordField() owner = fields.NestedField(properties={ 'id': fields.KeywordField(), 'full_name': fields.TextField() }) initiative = fields.NestedField(properties={ 'title': fields.TextField(), 'pitch': fields.TextField(), 'story': fields.TextField(), }) theme = fields.NestedField( attr='initiative.theme', properties={ 'id': fields.KeywordField(), } ) categories = fields.NestedField( attr='initiative.categories', properties={ 'id': fields.LongField(), 'slug': fields.KeywordField(), } ) position = fields.GeoPointField() country = fields.KeywordField() expertise = fields.NestedField( properties={ 'id': fields.KeywordField(), } ) segments = fields.NestedField( properties={ 'id': fields.KeywordField(), 'type': fields.KeywordField(attr='type.slug'), 'name': fields.TextField() } ) location = fields.NestedField( attr='location', properties={ 'id': fields.LongField(), 'formatted_address': fields.TextField(), } ) initiative_location = fields.NestedField( attr='initiative.location', properties={ 'id': fields.LongField(), 'name': fields.TextField(), 'city': fields.TextField(), } ) contributions = fields.DateField() contribution_count = fields.IntegerField() activity_date = fields.DateField() class Meta(object): model = Activity def get_queryset(self): return super(ActivityDocument, self).get_queryset().select_related( 'initiative', 'owner', ) @classmethod def search(cls, using=None, index=None): # Use search class that supports polymorphic models return Search( using=using or cls._doc_type.using, index=index or cls._doc_type.index, doc_type=[cls], model=cls._doc_type.model ) def prepare_contributions(self, instance): return [ contribution.created for contribution in instance.contributions.filter(status__in=('new', 'success')) ] def prepare_type(self, instance): return str(instance.__class__.__name__.lower()) def prepare_contribution_count(self, instance): return len(instance.contributions.filter(status__in=('new', 'success'))) def prepare_country(self, instance): if instance.initiative.location: return instance.initiative.location.country_id if instance.initiative.place: return instance.initiative.place.country_id def prepare_location(self, instance): if hasattr(instance, 'location') and instance.location: return { 'id': instance.location.pk, 'formatted_address': instance.location.formatted_address } def prepare_expertise(self, instance): if hasattr(instance, 'expertise') and instance.expertise: return {'id': instance.expertise_id} def prepare_position(self, instance): return None def prepare_deadline(self, instance): return None def prepare_date(self, instance): return None
class CollectionItemDocument(DocType): """Collection item document.""" # ID id = fields.IntegerField(attr='id') record_number = KeywordField() inventory_number = KeywordField() api_url = KeywordField(index="not_analyzed") web_url = KeywordField(index="not_analyzed") # ******************************************************************** # *************** Main data fields for search and filtering ********** # ******************************************************************** importer_uid = KeywordField(attr='importer_uid_indexing') language_code_orig = KeywordField(attr='language_code_orig') department = StringField( attr='department_indexing', analyzer=html_strip, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), # 'suggest': fields.CompletionField(), }) # ******************************************************************** # ***************************** English ****************************** # ******************************************************************** title_en = StringField( attr='title_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), # 'suggest': fields.CompletionField(), }) description_en = StringField( attr='description_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), # 'suggest': fields.CompletionField(), }) period_en = StringField( attr='period_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), # 'suggest': fields.CompletionField(), }) period_1_en = fields.NestedField( attr='period_1_en_indexing', properties={ 'name': StringField(analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), }), 'period_2_en': fields.NestedField( properties={ 'name': StringField(analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), }), 'period_3_en': fields.NestedField( properties={ 'name': StringField(analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), }), 'period_4_en': fields.NestedField( properties={ 'name': StringField( analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), }) }) }) }) }) primary_object_type_en = StringField( attr='primary_object_type_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), 'suggest': fields.CompletionField(), }) object_type_en = StringField(attr='object_type_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), 'suggest': fields.CompletionField(), }) # To be shown on the detail page object_type_detail_en = fields.TextField( attr='object_type_detail_en_indexing', index='no') material_en = StringField( attr='material_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), # 'suggest': fields.CompletionField(), }) # To be shown on the detail page material_detail_en = fields.TextField(attr='material_detail_en_indexing', index='no') city_en = StringField( attr='city_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), # 'suggest': fields.CompletionField(), }) country_en = StringField( attr='country_en_indexing', analyzer=html_strip_synonyms_en, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='english'), # 'suggest': fields.CompletionField(), }) # To be shown on the detail page references_en = fields.TextField(attr='references_en_indexing', index='no') # To be shown on the detail page acquired_en = fields.TextField(attr='acquired_en_indexing', index='no') # To be shown on the detail page site_found_en = fields.TextField(attr='site_found_en_indexing', index='no') # To be shown on the detail page reign_en = fields.TextField(attr='reign_en_indexing', index='no') # To be shown on the detail page keywords_en = fields.TextField(attr='keywords_en_indexing', index='no') # To be shown on the detail page dynasty_en = fields.TextField(attr='dynasty_en_indexing', index='no') # New fields # To be shown on the detail page credit_line_en = fields.TextField(attr='credit_line_en_indexing', index='no') # To be shown on the detail page region_en = fields.TextField(attr='region_en_indexing', index='no') # To be shown on the detail page sub_region_en = fields.TextField(attr='sub_region_en_indexing', index='no') # To be shown on the detail page locale_en = fields.TextField(attr='locale_en_indexing', index='no') # To be shown on the detail page excavation_en = fields.TextField(attr='excavation_en_indexing', index='no') # To be shown on the detail page museum_collection_en = fields.TextField( attr='museum_collection_en_indexing', index='no') # To be shown on the detail page style_en = fields.TextField(attr='style_en_indexing', index='no') # To be shown on the detail page culture_en = fields.TextField(attr='culture_en_indexing', index='no') # To be shown on the detail page inscriptions_en = fields.TextField(attr='inscriptions_en_indexing', index='no') # To be shown on the detail page provenance_en = fields.TextField(attr='provenance_en_indexing', index='no') # To be shown on the detail page exhibitions_en = fields.TextField(attr='exhibitions_en_indexing', index='no') # ******************************************************************** # ****************************** Dutch ******************************* # ******************************************************************** title_nl = StringField( attr='title_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), # 'suggest': fields.CompletionField(), }) description_nl = StringField( attr='description_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), # 'suggest': fields.CompletionField(), }) period_nl = StringField( attr='period_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), # 'suggest': fields.CompletionField(), }) period_1_nl = fields.NestedField( attr='period_1_nl_indexing', properties={ 'name': StringField(analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), }), 'period_2_nl': fields.NestedField( properties={ 'name': StringField(analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), }), 'period_3_nl': fields.NestedField( properties={ 'name': StringField(analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), }), 'period_4_nl': fields.NestedField( properties={ 'name': StringField( analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), }) }) }) }) }) primary_object_type_nl = StringField( attr='primary_object_type_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), 'suggest': fields.CompletionField(), }) object_type_nl = StringField(attr='object_type_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), 'suggest': fields.CompletionField(), }) # To be shown on the detail page object_type_detail_nl = fields.TextField( attr='object_type_detail_nl_indexing', index='no') material_nl = StringField( attr='material_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), # 'suggest': fields.CompletionField(), }) # To be shown on the detail page material_detail_nl = fields.TextField(attr='material_detail_nl_indexing', index='no') city_nl = StringField( attr='city_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), # 'suggest': fields.CompletionField(), }) country_nl = StringField( attr='country_nl_indexing', analyzer=html_strip_synonyms_nl, fields={ 'raw': KeywordField(), 'natural': StringField(analyzer='dutch'), # 'suggest': fields.CompletionField(), }) # To be shown on the detail page keywords_nl = fields.TextField(attr='keywords_nl_indexing', index='no') # To be shown on the detail page acquired_nl = fields.TextField(attr='acquired_nl_indexing', index='no') # To be shown on the detail page site_found_nl = fields.TextField(attr='site_found_nl_indexing', index='no') # To be shown on the detail page reign_nl = fields.TextField(attr='reign_nl_indexing', index='no') # To be shown on the detail page references_nl = fields.TextField(attr='references_nl_indexing', index='no') # To be shown on the detail page dynasty_nl = fields.TextField(attr='dynasty_nl_indexing', index='no') # New fields # To be shown on the detail page credit_line_nl = fields.TextField(attr='credit_line_nl_indexing', index='no') # To be shown on the detail page region_nl = fields.TextField(attr='region_nl_indexing', index='no') # To be shown on the detail page sub_region_nl = fields.TextField(attr='sub_region_nl_indexing', index='no') # To be shown on the detail page locale_nl = fields.TextField(attr='locale_nl_indexing', index='no') # To be shown on the detail page excavation_nl = fields.TextField(attr='excavation_nl_indexing', index='no') # To be shown on the detail page museum_collection_nl = fields.TextField( attr='museum_collection_nl_indexing', index='no') # To be shown on the detail page style_nl = fields.TextField(attr='style_nl_indexing', index='no') # To be shown on the detail page culture_nl = fields.TextField(attr='culture_nl_indexing', index='no') # To be shown on the detail page inscriptions_nl = fields.TextField(attr='inscriptions_nl_indexing', index='no') # To be shown on the detail page provenance_nl = fields.TextField(attr='provenance_nl_indexing', index='no') # To be shown on the detail page exhibitions_nl = fields.TextField(attr='exhibitions_nl_indexing', index='no') # ******************************************************************** # ************************** Language independent ******************** # ******************************************************************** dimensions = StringField( attr='dimensions_indexing', analyzer=html_strip, fields={ 'raw': KeywordField(), 'natural': StringField(), # 'suggest': fields.CompletionField(), }) object_date_begin = StringField( attr='object_date_begin_indexing', analyzer=html_strip, fields={ 'raw': KeywordField(), 'natural': StringField(), # 'suggest': fields.CompletionField(), }) object_date_end = StringField( attr='object_date_end_indexing', analyzer=html_strip, fields={ 'raw': KeywordField(), 'natural': StringField(), # 'suggest': fields.CompletionField(), }) location = fields.GeoPointField(attr='geo_location_indexing') # List of 32x32 PNG versions of the images. Full path to. images = fields.ListField(StringField(attr='images_indexing')) # List of image URLs. images_urls = fields.ListField( fields.ObjectField(attr='images_urls_indexing', properties={ 'th': KeywordField(index="not_analyzed"), 'lr': KeywordField(index="not_analyzed"), })) # Classified as by our AI classified_as = fields.ListField( StringField(attr='classified_as_indexing', fields={ 'raw': KeywordField(), })) # Classified as 1st element classified_as_1 = StringField(attr='classified_as_1_indexing', fields={ 'raw': KeywordField(), }) # Classified as 2nd element classified_as_2 = StringField(attr='classified_as_2_indexing', fields={ 'raw': KeywordField(), }) # Classified as 3rd element classified_as_3 = StringField(attr='classified_as_3_indexing', fields={ 'raw': KeywordField(), }) # ******************************************************************** # ************** Nested fields for search and filtering ************** # ******************************************************************** # # City object # country = fields.NestedField( # properties={ # 'name': StringField( # analyzer=html_strip, # fields={ # 'raw': KeywordField(), # 'suggest': fields.CompletionField(), # } # ), # 'info': StringField(analyzer=html_strip), # 'location': fields.GeoPointField(attr='location_field_indexing'), # } # ) # # location = fields.GeoPointField(attr='location_field_indexing') class Meta(object): """Meta options.""" model = Item # The model associate with this DocType def get_queryset(self): """Filter out items that are not eligible for indexing.""" qs = super(CollectionItemDocument, self).get_queryset() # qs = qs.select_related('period_node').prefetch_related('images') filters = [] for field in ['title']: for language in ['en', 'nl']: filters.extend([ Q(**{"{}_{}__isnull".format(field, language): True}), Q(**{"{}_{}__exact".format(field, language): ''}), ]) if filters: qs = qs.exclude(six.moves.reduce(operator.or_, filters)) # We concatenate ``object_type`` and ``classification`` fields, after # cleaning them. Therefore, db-only checks don't work here. ids = [] for item in qs: if not (item.object_type_en_indexing and item.object_type_nl_indexing): ids.append(item.pk) return qs.exclude(id__in=ids) def prepare_department(self, instance): """Prepare department.""" return instance.department_indexing \ if instance.department_indexing\ else VALUE_NOT_SPECIFIED def prepare_object_date_begin(self, instance): """Prepare material.""" return instance.object_date_begin_indexing def prepare_object_date_end(self, instance): """Prepare material.""" return instance.object_date_end_indexing # ******************************************************************** # ***************************** English ****************************** # ******************************************************************** def prepare_material_en(self, instance): """Prepare material.""" return instance.material_en_indexing \ if instance.material_en_indexing\ else VALUE_NOT_SPECIFIED def prepare_period_en(self, instance): """Prepare state.""" return instance.period_en_indexing \ if instance.period_en_indexing \ else VALUE_NOT_SPECIFIED def prepare_dynasty_en(self, instance): """Prepare dynasty.""" return instance.dynasty_en_indexing \ if instance.dynasty_en_indexing \ else VALUE_NOT_SPECIFIED def prepare_description_en(self, instance): """Prepare description.""" return instance.description_en_indexing \ if instance.description_en_indexing\ else VALUE_NOT_SPECIFIED def prepare_city_en(self, instance): """Prepare city.""" return instance.city_en_indexing \ if instance.city_en_indexing\ else VALUE_NOT_SPECIFIED def prepare_country_en(self, instance): """Prepare country.""" return instance.country_en_indexing \ if instance.country_en_indexing \ else VALUE_NOT_SPECIFIED # ******************************************************************** # ****************************** Dutch ******************************* # ******************************************************************** def prepare_material_nl(self, instance): """Prepare material.""" return instance.material_nl_indexing \ if instance.material_nl_indexing\ else VALUE_NOT_SPECIFIED def prepare_period_nl(self, instance): """Prepare state.""" return instance.period_nl_indexing \ if instance.period_nl_indexing \ else VALUE_NOT_SPECIFIED def prepare_dynasty_nl(self, instance): """Prepare dynasty.""" return instance.dynasty_nl_indexing \ if instance.dynasty_nl_indexing \ else VALUE_NOT_SPECIFIED def prepare_description_nl(self, instance): """Prepare description.""" return instance.description_nl_indexing \ if instance.description_nl_indexing\ else VALUE_NOT_SPECIFIED def prepare_city_nl(self, instance): """Prepare city.""" return instance.city_nl_indexing \ if instance.city_nl_indexing\ else VALUE_NOT_SPECIFIED def prepare_country_nl(self, instance): """Prepare country.""" return instance.country_nl_indexing \ if instance.country_nl_indexing \ else VALUE_NOT_SPECIFIED
class LocationDocument(DocType): """ Location document. """ full = fields.StringField( analyzer=html_strip, fields={ "raw": KeywordField(), "suggest": fields.CompletionField(), "context": fields.CompletionField( contexts=[ { "name": "category", "type": "category", "path": "category.raw", }, { "name": "occupied", "type": "category", "path": "occupied.raw", }, ] ), # edge_ngram_completion "q": StringField( analyzer=edge_ngram_completion ), } ) partial = fields.StringField( analyzer=html_strip, fields={ "raw": KeywordField(), "suggest": fields.CompletionField(), "context": fields.CompletionField( contexts=[ { "name": "category", "type": "category", "path": "category.raw", }, { "name": "occupied", "type": "category", "path": "occupied.raw", }, ] ), # edge_ngram_completion "q": StringField( analyzer=edge_ngram_completion ), } ) postcode = fields.StringField( analyzer=html_strip, fields={ "raw": KeywordField(), "suggest": fields.CompletionField(), "context": fields.CompletionField( contexts=[ { "name": "category", "type": "category", "path": "category.raw", }, { "name": "occupied", "type": "category", "path": "occupied.raw", }, ] ), } ) number = fields.StringField( attr="address_no", analyzer=html_strip, fields={ "raw": KeywordField(), } ) address = fields.StringField( attr="address_street", analyzer=html_strip, fields={ "raw": KeywordField(), } ) town = fields.StringField( attr="address_town", analyzer=html_strip, fields={ "raw": KeywordField(), } ) authority = fields.StringField( attr="authority_name", analyzer=html_strip, fields={ "raw": KeywordField(), } ) # URL fields /geocode/slug geocode = fields.StringField( analyzer=html_strip, fields={ "raw": KeywordField(), } ) slug = fields.StringField( analyzer=html_strip, fields={ "raw": KeywordField(), } ) # Filter fields category = fields.StringField( attr="group", analyzer=html_strip, fields={ "raw": KeywordField(), } ) occupied = fields.StringField( attr="occupation_status_text", analyzer=html_strip, fields={ "raw": KeywordField(), } ) size = fields.FloatField(attr="floor_area") staff = fields.FloatField(attr="employee_count") rent = fields.FloatField(attr="rental_valuation") revenue = fields.FloatField(attr="revenue") coordinates = fields.GeoPointField(attr="location_field_indexing") class Meta(object): """Meta options.""" model = Location # The model associate with this DocType parallel_indexing = True queryset_pagination = 50 # This will split the queryset
class DeviceDocument(DocType): date_modified = fields.DateField() user = fields.ObjectField(properties={ 'mobile': fields.StringField(), 'email': fields.StringField(), }) location = fields.GeoPointField() tag_suggest = fields.CompletionField() stores = fields.NestedField(properties={ 'store_id': fields.StringField(), 'date': fields.DateField(), }) '''offers = fields.NestedField(properties={ 'reward': fields.StringField(), 'date_redeemed': fields.DateField(), 'expiry_date': fields.DateField(), 'date_claimed': fields.DateField(), })''' class Meta: model = Device # The model associated with this DocType # The fields of the model you want to be indexed in Elasticsearch fields = [ 'device_id', 'date_created', 'active', ] related_models = [User] # Ignore auto updating of Elasticsearch when a model is saved # or deleted: # ignore_signals = True # Don't perform an index refresh after every update (overrides global setting): # auto_refresh = False # Paginate the django queryset used to populate the index with the specified size # (by default there is no pagination) # queryset_pagination = 5000 def get_instances_from_related(self, related_instance): """If related_models is set, define how to retrieve the Car instances from the related model.""" return related_instance.device_set.all() def update_stores(self, store_id): # https://github.com/istresearch/kibana-object-format self.stores.append({'store_id': store_id, 'date': datetime.now()}) self.save() def prepare_stores(self, instance): #print instance.stores try: device = self.get(id=instance.pk) return device.stores if device.stores else [] except TransportError: return [] def prepare_offers(self, instance): return [] def prepare_tag_suggest(self, instance): return [ "red", "tiny", ] def prepare_location(self, instance): return dict(lat=10.0089331, lon=76.3155752) def prepare_date_modified(self, instance): return instance.date_modified
class PublicSource(PublicSourceBase): id = fields.KeywordField(attr='id') name = fields.TextField( attr='name', fields={'keyword': fields.KeywordField( normalizer=basic_normalizer )}, index_prefixes={} ) type = fields.IntegerField(attr='type.id') is_private = fields.BooleanField(attr='is_private') is_public = fields.BooleanField() parent = fields.TextField( attr='parent.name', fields={'keyword': fields.KeywordField( normalizer=basic_normalizer )}, index_prefixes={} ) attributes = fields.ObjectField() collections = fields.ObjectField() credits = fields.ObjectField() geo_location = fields.GeoPointField() class Index: name = 'public_sources' class Django: model = Source def get_queryset(self): return Source.objects.filter(type=13, workflow__is_public=True) def prepare_is_public(self, instance): try: return instance.workflow.is_public except ObjectDoesNotExist: return False def prepare_attributes(self, instance): attribute_list = [] dates_list = [] for attribute in instance.attributes.all(): label = attribute.attribute_type.short_name if attribute.attribute_type.data_type == 'DATE': if attribute.value_DATE: dates_list.append(attribute.value_DATE.strftime('%Y-%m-%d')) elif attribute.value_DATE_y: d = attribute.value_DATE_d or 1 m = attribute.value_DATE_m or 1 y = attribute.value_DATE_y dates_list.append(date(y, m, d).strftime('%Y-%m-%d')) elif attribute.attribute_type.data_type == 'TXT': attribute_list.append((label, attribute.value_TXT)) else: attribute_list.append((label, str(attribute))) if dates_list: attribute_list.append(('date', dates_list)) if attribute_list: return dict(attribute_list) def prepare_collections(self, instance): try: if instance.sets.all().count() > 0: cols = [('name', set.set_id.name) for set in instance.sets.all() if set.set_id.set_type == 2] if cols: return dict(cols) else: return [{'name': 'none'}] except ObjectDoesNotExist: return [{'name': 'none'}] def prepare_credits(self, instance): credit_list = [] for credit in instance.credits.all(): agent = credit.agent.standard_name type = credit.get_type_display() credit_list.append({ 'agent': f'{agent} ({type})', 'type': type }) return credit_list def prepare_geo_location(self, instance): locales = instance.attributes.filter(attribute_type=36) if locales.exists(): loc_id = locales[0].value_JSON['id'] locale = LocaleReference.objects.get(id=loc_id) return f'{locale.latitude},{locale.longitude}'
class AddressDocument(Document): """Address Elasticsearch document.""" # In different parts of the code different fields are used. There are # a couple of use cases: (1) more-like-this functionality, where `title`, # `description` and `summary` fields are used, (2) search and filtering # functionality where all of the fields are used. # ID id = fields.IntegerField(attr='id') # ******************************************************************** # *********************** Main data fields for search **************** # ******************************************************************** __street_fields = { 'raw': KeywordField(), 'suggest': fields.CompletionField(), } if ELASTICSEARCH_GTE_5_0: __street_fields.update( { 'suggest_context': fields.CompletionField( contexts=[ { "name": "loc", "type": "geo", "path": "location", "precision": "1000km", }, ] ), } ) street = StringField( analyzer=html_strip, fields=__street_fields ) house_number = StringField(analyzer=html_strip) appendix = StringField(analyzer=html_strip) zip_code = StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), } ) # ******************************************************************** # ********** Additional fields for search and filtering ************** # ******************************************************************** # City object city = fields.ObjectField( properties={ 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), } ), 'info': StringField(analyzer=html_strip), 'location': fields.GeoPointField(attr='location_field_indexing'), 'country': fields.ObjectField( properties={ 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), } ), 'info': StringField(analyzer=html_strip), 'location': fields.GeoPointField( attr='location_field_indexing' ) } ) } ) # Country object country = fields.NestedField( attr='country_indexing', properties={ 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), } ), 'city': fields.ObjectField( properties={ 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), }, ), }, ), }, ) # Continent object continent = fields.NestedField( attr='continent_indexing', properties={ 'id': fields.IntegerField(), 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), } ), 'country': fields.NestedField( properties={ 'id': fields.IntegerField(), 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), } ), 'city': fields.NestedField( properties={ 'id': fields.IntegerField(), 'name': StringField( analyzer=html_strip, fields={ 'raw': KeywordField(), } ) } ) } ) } ) location = fields.GeoPointField( attr='location_field_indexing', ) class Django(object): model = Address # The model associate with this Document class Meta(object): parallel_indexing = True
class AddressDocument(Document): id = fields.IntegerField(attr='id') street = StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }) house_number = StringField(analyzer=html_strip) appendix = StringField(analyzer=html_strip) zip_code = StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }) city = fields.ObjectField( properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField() }), 'info': StringField(analyzer=html_strip), 'location': fields.GeoPointField(attr='location_field_indexing'), 'country': fields.ObjectField( properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }), 'info': StringField(analyzer=html_strip), 'location': fields.GeoPointField(attr='location_field_indexing') }) }) # Defining the ``@property`` functions in the address model country = fields.NestedField( attr='country_indexing', properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField(), }), 'city': fields.ObjectField( properties={ 'name': StringField(analyzer=html_strip, fields={'raw': KeywordField()}) }) }) continent = fields.NestedField( attr='continent_indexing', properties={ 'name': StringField(analyzer=html_strip, fields={ 'raw': KeywordField(), 'suggest': fields.CompletionField() }), 'country': fields.NestedField( properties={ 'name': StringField(analyzer=html_strip, fields={'raw': KeywordField()}), 'city': fields.NestedField( properties={ 'name': StringField(analyzer=html_strip, fields={'raw': KeywordField()}) }) }) }) location = fields.GeoPointField(attr='location_field_indexing') class Django(object): model = Address