class BookDocument(DocType):
"""Book Elasticsearch document."""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
__title_fields = {
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion
),
'mlt': StringField(analyzer='english'),
}
if ELASTICSEARCH_GTE_5_0:
__title_fields.update(
{
'suggest_context': fields.CompletionField(
contexts=[
{
"name": "tag",
"type": "category",
"path": "tags.raw",
},
{
"name": "state",
"type": "category",
"path": "state.raw",
},
{
"name": "publisher",
"type": "category",
"path": "publisher.raw",
},
]
),
}
)
title = StringField(
analyzer=html_strip,
fields=__title_fields
)
description = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'mlt': StringField(analyzer='english'),
}
)
summary = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'mlt': StringField(analyzer='english'),
}
)
# ********************************************************************
# ********** Additional fields for search and filtering **************
# ********************************************************************
authors = fields.ListField(
StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
)
# Publisher
publisher = StringField(
attr='publisher_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
)
# Publication date
publication_date = fields.DateField()
# State
state = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
# ISBN
isbn = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
# Price
price = fields.FloatField()
# Pages
pages = fields.IntegerField()
# Stock count
stock_count = fields.IntegerField()
# Tags
tags = StringField(
attr='tags_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(multi=True),
'suggest': fields.CompletionField(multi=True),
},
multi=True
)
# Date created
created = fields.DateField()
null_field = StringField(attr='null_field_indexing')
class Meta(object):
"""Meta options."""
model = Book # The model associate with this DocType
parallel_indexing = True
# queryset_pagination = 500000 # This will split the queryset
# # into parts while indexing
def prepare_summary(self, instance):
"""Prepare summary."""
return instance.summary[:32766] if instance.summary else None
def prepare_authors(self, instance):
"""Prepare authors."""
return [author.name for author in instance.authors.all()]
class LocationDocument(DocType):
"""
Location document.
"""
full = fields.StringField(
analyzer=html_strip,
fields={
"raw":
KeywordField(),
"suggest":
fields.CompletionField(),
"context":
fields.CompletionField(contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]),
# edge_ngram_completion
"q":
StringField(analyzer=edge_ngram_completion),
})
partial = fields.StringField(
analyzer=html_strip,
fields={
"raw":
KeywordField(),
"suggest":
fields.CompletionField(),
"context":
fields.CompletionField(contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]),
# edge_ngram_completion
"q":
StringField(analyzer=edge_ngram_completion),
})
postcode = fields.StringField(analyzer=html_strip,
fields={
"raw":
KeywordField(),
"suggest":
fields.CompletionField(),
"context":
fields.CompletionField(contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]),
})
number = fields.StringField(attr="address_no",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
address = fields.StringField(attr="address_street",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
town = fields.StringField(attr="address_town",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
authority = fields.StringField(attr="authority_name",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# URL fields /geocode/slug
geocode = fields.StringField(analyzer=html_strip,
fields={
"raw": KeywordField(),
})
slug = fields.StringField(analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# Filter fields
category = fields.StringField(attr="group",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
occupied = fields.StringField(attr="occupation_status_text",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
size = fields.FloatField(attr="floor_area")
staff = fields.FloatField(attr="employee_count")
rent = fields.FloatField(attr="rental_valuation")
revenue = fields.FloatField(attr="revenue")
coordinates = fields.GeoPointField(attr="location_field_indexing")
class Meta(object):
"""Meta options."""
model = Location # The model associate with this DocType
parallel_indexing = True
queryset_pagination = 1000 # This will split the queryset
class JournalDocument(Document):
"""Journal Elasticsearch document."""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ISBN/ID
isbn = StringField(analyzer=html_strip, fields={
'raw': KeywordField(),
})
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
title = StringField(analyzer=html_strip,
fields={
'raw':
KeywordField(),
'suggest':
fields.CompletionField(),
'edge_ngram_completion':
StringField(analyzer=edge_ngram_completion),
'mlt':
StringField(analyzer='english'),
})
description = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'mlt': StringField(analyzer='english'),
})
summary = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'mlt': StringField(analyzer='english'),
})
# ********************************************************************
# ********** Additional fields for search and filtering **************
# ********************************************************************
# Publication date
publication_date = fields.DateField()
# Price
price = fields.FloatField()
# Pages
pages = fields.IntegerField()
# Stock count
stock_count = fields.IntegerField()
# Date created
created = fields.DateField(attr='created_indexing')
class Django(object):
model = Journal # The model associate with this Document
class Meta:
parallel_indexing = True
# queryset_pagination = 50 # This will split the queryset
# # into parts while indexing
def prepare_summary(self, instance):
"""Prepare summary."""
return instance.summary[:32766] if instance.summary else None
class AddressDocument(Document):
id = fields.IntegerField(attr='id')
street = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
house_number = StringField(analyzer=html_strip)
appendix = StringField(analyzer=html_strip)
zip_code = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
city = fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField()
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing'),
'country':
fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing')
})
})
# Defining the ``@property`` functions in the address model
country = fields.NestedField(
attr='country_indexing',
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'city':
fields.ObjectField(
properties={
'name':
StringField(analyzer=html_strip,
fields={'raw': KeywordField()})
})
})
continent = fields.NestedField(
attr='continent_indexing',
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField()
}),
'country':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip,
fields={'raw': KeywordField()}),
'city':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip,
fields={'raw': KeywordField()})
})
})
})
location = fields.GeoPointField(attr='location_field_indexing')
class Django(object):
model = Address
class ResourceDocument(Document):
meta = fields.KeywordField()
content = fields.TextField(attr="title.main_title", store=True)
is_private = fields.BooleanField()
title = fields.TextField(
analyzer=text_folding_analyzer,
fields={
"raw": fields.KeywordField(),
"sort": fields.KeywordField(normalizer=lowercase_sort_normalizer),
"suggest": fields.CompletionField(),
},
**copy_to_content,
)
form_genre = get_controlled_term_field(options=copy_to_content)
subjects = get_controlled_term_field(options=copy_to_content)
date_display = fields.TextField(**copy_to_content)
year = fields.IntegerField(**copy_to_content)
summary = fields.TextField(**copy_to_content)
classifications_printing_publishing = fields.ObjectField(
properties={
"edition": get_controlled_term_field(options=copy_to_content)
})
classifications_translation = fields.ObjectField(
properties={
"edition": get_controlled_term_field(options=copy_to_content)
})
classifications_paratext = fields.ObjectField(
properties={
"edition": get_controlled_term_field(options=copy_to_content)
})
classifications_paratext_functions = fields.ObjectField(
properties={
"edition": get_controlled_term_field(options=copy_to_content)
})
contributions = fields.ObjectField(
properties={
"agent": get_agent_field(options=copy_to_content),
"roles": get_controlled_term_field(),
})
published_as = fields.KeywordField()
languages = get_controlled_term_field(options=copy_to_content)
places = fields.ObjectField(
properties={
"place":
get_place_field(options=copy_to_content),
"fictional_place":
fields.TextField(fields={"raw": fields.KeywordField()}),
})
relationships = fields.ObjectField(
properties={
"relationship_type": get_controlled_term_field(
options=copy_to_content),
"related_to": get_resource_field(options=copy_to_content),
})
# events = get_event_field(options=copy_to_content)
is_original = fields.BooleanField()
is_translation = fields.BooleanField()
has_date_radical = fields.KeywordField()
authors = fields.ObjectField(
attr="get_authors_source_text",
properties={"person": get_agent_field(options=copy_to_content)},
)
translated_from = get_controlled_term_field(options=copy_to_content)
class Index:
name = "rt-resources"
class Django:
model = Resource
fields = ["id"]
related_models = [
Classification,
Contribution,
ControlledTerm,
Date,
ResourceLanguage,
ResourcePlace,
ResourceRelationship,
Title,
]
def get_queryset(self):
return (super().get_queryset().exclude(
relationships__relationship_type__label="paratext of").
select_related("title", "date"))
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, Date):
return related_instance.resource
if isinstance(related_instance, (ControlledTerm, Title)):
return related_instance.resources.all()
if isinstance(
related_instance,
(
Classification,
Contribution,
ResourceLanguage,
ResourcePlace,
ResourceRelationship,
),
):
return related_instance.resource
def prepare_meta(self, instance):
if instance.is_original():
return "source texts"
meta = []
if instance.get_paratext():
meta.append("paratexts")
if instance.is_translation():
meta.append("translations")
return meta
def prepare_title(self, instance):
titles = [str(instance.title)]
for relationship in instance.get_paratext():
paratext = relationship.resource
if str(paratext.title) != str(instance.title):
titles.append(str(paratext.title))
return titles
def prepare_form_genre(self, instance):
return self._get_subjects(instance, ["fast-forms", "rt-agt"])
def _get_subjects(self, instance, prefix):
subjects = [{
"label": item.label
} for item in instance.subjects.filter(vocabulary__prefix__in=prefix)]
for relationship in instance.get_paratext():
subjects.extend(self._get_subjects(relationship.resource, prefix))
if subjects:
subjects.append({"label": "any"})
return subjects
def prepare_subjects(self, instance):
return self._get_subjects(instance, ["fast-topic"])
def prepare_date_display(self, instance):
resource = self._get_resource(instance)
if resource.date:
return str(resource.date)
def _get_resource(self, resource):
if resource.is_paratext():
return resource.paratext_of()
return resource
def prepare_year(self, instance):
resource = self._get_resource(instance)
if resource.date:
date_earliest = resource.date.get_date_earliest()
date_latest = resource.date.get_date_latest()
if date_earliest and date_latest:
return [
year
for year in range(date_earliest.year, date_latest.year + 1)
]
if date_earliest:
return date_earliest.year
if date_latest:
return date_latest.year
def prepare_summary(self, instance):
summaries = []
if instance.summary:
summaries = [instance.summary]
for relationship in instance.get_paratext():
if relationship.resource.summary:
summaries.append(relationship.resource.summary)
return summaries
def prepare_classifications_printing_publishing(self, instance):
return self._get_classifications(instance, "rt-ppt")
def _get_classifications(self, instance, prefix):
classifications = [
{
"edition": {
"label": item.edition.label
},
} for item in instance.classifications.filter(
edition__vocabulary__prefix=prefix)
if item.edition.label.lower() not in ["original", "source-text"]
]
for relationship in instance.get_paratext():
classifications.extend(
self._get_classifications(relationship.resource, prefix))
if classifications:
classifications.append({"edition": {"label": "any"}})
return classifications
def prepare_classifications_translation(self, instance):
return self._get_classifications(instance, "rt-tt")
def prepare_classifications_paratext(self, instance):
return self._get_classifications(instance, "rt-pt")
def prepare_classifications_paratext_functions(self, instance):
return self._get_classifications(instance, "rt-ptf")
def prepare_contributions(self, instance):
contributions = [{
"agent": {
"id":
item.agent.id,
"name":
"Anonymous" if item.agent.name.startswith("Anon") else
item.agent.get_index_name(),
},
"roles": [{
"label": f"{role.label} of translation paratext"
if item.resource.is_paratext() else role.label
for role in item.roles.all()
}],
} for item in instance.get_contributions(include_paratext=True)]
if contributions:
contributions.append({
"agent": {
"name": "any"
},
"roles": [{
"label": "any"
}]
})
return contributions
def prepare_published_as(self, instance):
published_as = []
for item in instance.get_contributions(include_paratext=True):
if item.published_as:
published_as.append(item.published_as)
if published_as:
published_as.append("any")
return published_as
def prepare_languages(self, instance):
languages = [{
"label": item.language.label
} for item in instance.languages.all()]
for relationship in instance.get_paratext():
languages.extend(self.prepare_languages(relationship.resource))
if languages:
languages.append({"label": "any"})
return languages
def prepare_places(self, instance):
places = []
for item in instance.places.all():
address = ""
place = {}
if item.fictional_place:
address = item.fictional_place
place = {
"fictional_place": item.fictional_place,
"place": {
"address": address
},
}
if item.place:
address = (f"{address} ({item.place.address})"
if address else item.place.address)
place["place"] = {
"address": address,
"geo": item.place.geo,
"country": {
"name": item.place.country.name
},
}
places.append(place)
if places:
places.append(
{"place": {
"address": "any",
"country": {
"name": "any"
}
}})
return places
def prepare_relationships(self, instance):
relationships = [{
"relationship_type": {
"label": item.relationship_type.label
},
"related_to": {
"id": item.related_to.id,
"title": {
"main_title": str(item.related_to.title)
},
},
} for item in instance.relationships.all()]
if relationships:
relationships.append({"relationship_type": {"label": "any"}})
return relationships
def prepare_events(self, instance):
events = [{
"id": item.id,
"title": item.title,
"place": {
"address": item.place.address,
"country": {
"name": item.place.country.name
},
},
} for item in instance.events.all()]
if events:
events.append({
"title": "any",
"place": {
"address": "any",
"country": {
"name": "any"
}
},
})
return events
def prepare_has_date_radical(self, instance):
if instance.has_date_radical():
return "yes"
return "no"
def prepare_translated_from(self, instance):
languages = []
if instance.get_languages_source_text():
languages = [{
"label": language.label
} for language in instance.get_languages_source_text()]
if languages:
languages.append({"label": "any"})
return languages
class LocationDocument(Document):
"""
Location document.
"""
# Full fields
__full_fields = {
"raw": KeywordField(),
# edge_ngram_completion
"q": StringField(analyzer=edge_ngram_completion),
}
if ELASTICSEARCH_GTE_5_0:
__full_fields.update({
"suggest":
fields.CompletionField(),
"context":
fields.CompletionField(contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]),
})
full = StringField(analyzer=html_strip, fields=__full_fields)
# Partial fields
__partial_fields = {
"raw": KeywordField(),
# edge_ngram_completion
"q": StringField(analyzer=edge_ngram_completion),
}
if ELASTICSEARCH_GTE_5_0:
__partial_fields.update({
"suggest":
fields.CompletionField(),
"context":
fields.CompletionField(contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]),
})
partial = StringField(analyzer=html_strip, fields=__partial_fields)
# Postcode
__postcode_fields = {
"raw": KeywordField(),
}
if ELASTICSEARCH_GTE_5_0:
__postcode_fields.update({
"suggest":
fields.CompletionField(),
"context":
fields.CompletionField(contexts=[
{
"name": "category",
"type": "category",
"path": "category.raw",
},
{
"name": "occupied",
"type": "category",
"path": "occupied.raw",
},
]),
})
postcode = StringField(analyzer=html_strip, fields=__postcode_fields)
# Number
number = StringField(attr="address_no",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# Address
address = StringField(attr="address_street",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# Town
town = StringField(attr="address_town",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# Authority
authority = StringField(attr="authority_name",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# URL fields /geocode/slug
geocode = StringField(analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# Slug
slug = StringField(analyzer=html_strip, fields={
"raw": KeywordField(),
})
# ********************* Filter fields **********************
# Category
category = StringField(attr="group",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
# Occupied
occupied = StringField(attr="occupation_status_text",
analyzer=html_strip,
fields={
"raw": KeywordField(),
})
size = fields.FloatField(attr="floor_area")
staff = fields.FloatField(attr="employee_count")
rent = fields.FloatField(attr="rental_valuation")
revenue = fields.FloatField(attr="revenue")
coordinates = fields.GeoPointField(attr="location_field_indexing")
class Django(object):
model = Location # The model associate with this Document
class Meta(object):
parallel_indexing = True
queryset_pagination = 50 # This will split the queryset
class AddressDocument(Document):
"""Address Elasticsearch document."""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
__street_fields = {
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
if ELASTICSEARCH_GTE_5_0:
__street_fields.update(
{
'suggest_context': fields.CompletionField(
contexts=[
{
"name": "loc",
"type": "geo",
"path": "location",
"precision": "1000km",
},
]
),
}
)
street = StringField(
analyzer=html_strip,
fields=__street_fields
)
house_number = StringField(analyzer=html_strip)
appendix = StringField(analyzer=html_strip)
zip_code = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
)
# ********************************************************************
# ********** Additional fields for search and filtering **************
# ********************************************************************
# City object
city = fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'info': StringField(analyzer=html_strip),
'location': fields.GeoPointField(attr='location_field_indexing'),
'country': fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'info': StringField(analyzer=html_strip),
'location': fields.GeoPointField(
attr='location_field_indexing'
)
}
)
}
)
# Country object
country = fields.NestedField(
attr='country_indexing',
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'city': fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
},
),
},
),
},
)
# Continent object
continent = fields.NestedField(
attr='continent_indexing',
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'country': fields.NestedField(
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
),
'city': fields.NestedField(
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
}
)
}
)
}
)
location = fields.GeoPointField(
attr='location_field_indexing',
)
class Django(object):
model = Address # The model associate with this Document
class Meta(object):
parallel_indexing = True
class CityDocument(Document):
"""City Elasticsearch document.
This document has been created purely for testing out complex fields.
"""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# ********************** Main data fields for search *****************
# ********************************************************************
name = StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
info = StringField(analyzer=html_strip)
# ********************************************************************
# ************** Nested fields for search and filtering **************
# ********************************************************************
# City object
country = fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'info':
StringField(analyzer=html_strip),
'location':
fields.GeoPointField(attr='location_field_indexing'),
})
location = fields.GeoPointField(attr='location_field_indexing')
# ********************************************************************
# ********** Other complex fields for search and filtering ***********
# ********************************************************************
boolean_list = fields.ListField(StringField(attr='boolean_list_indexing'))
# boolean_dict_indexing = fields.ObjectField(
# properties={
# 'true': fields.BooleanField(),
# 'false': fields.BooleanField(),
# }
# )
datetime_list = fields.ListField(
StringField(attr='datetime_list_indexing'))
# datetime_dict_indexing
float_list = fields.ListField(StringField(attr='float_list_indexing'))
# float_dict_indexing
integer_list = fields.ListField(StringField(attr='integer_list_indexing'))
# integer_dict_indexing
class Django(object):
model = City # The model associate with this Document
class Meta:
parallel_indexing = True
class BookDocument(DocType):
"""Book Elasticsearch document."""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
title = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
'edge_ngram_completion': StringField(
analyzer=edge_ngram_completion
),
'mlt': StringField(analyzer='english'),
}
)
description = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'mlt': StringField(analyzer='english'),
}
)
summary = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'mlt': StringField(analyzer='english'),
}
)
# ********************************************************************
# ********** Additional fields for search and filtering **************
# ********************************************************************
authors = fields.ListField(
StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
)
# Publisher
publisher = StringField(
attr='publisher_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
)
# Publication date
publication_date = fields.DateField()
# State
state = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
# ISBN
isbn = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
# Price
price = fields.FloatField()
# Pages
pages = fields.IntegerField()
# Stock count
stock_count = fields.IntegerField()
# Tags
tags = StringField(
attr='tags_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(multi=True),
'suggest': fields.CompletionField(multi=True),
},
multi=True
)
null_field = StringField(attr='null_field_indexing')
class Meta(object):
"""Meta options."""
model = Book # The model associate with this DocType
def prepare_summary(self, instance):
"""Prepare summary."""
return instance.summary[:32766]
def prepare_authors(self, instance):
"""Prepare authors."""
return [author.name for author in instance.authors.all()]
class CollectionItemDocument(DocType):
"""Collection item document."""
# ID
id = fields.IntegerField(attr='id')
record_number = KeywordField()
inventory_number = KeywordField()
api_url = KeywordField(index="not_analyzed")
web_url = KeywordField(index="not_analyzed")
# ********************************************************************
# *************** Main data fields for search and filtering **********
# ********************************************************************
importer_uid = KeywordField(attr='importer_uid_indexing')
language_code_orig = KeywordField(attr='language_code_orig')
department = StringField(
attr='department_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
# ********************************************************************
# ***************************** English ******************************
# ********************************************************************
title_en = StringField(
attr='title_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
description_en = StringField(
attr='description_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
period_en = StringField(
attr='period_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
period_1_en = fields.NestedField(
attr='period_1_en_indexing',
properties={
'name':
StringField(analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
}),
'period_2_en':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
}),
'period_3_en':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
}),
'period_4_en':
fields.NestedField(
properties={
'name':
StringField(
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
})
})
})
})
})
primary_object_type_en = StringField(
attr='primary_object_type_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
'suggest': fields.CompletionField(),
})
object_type_en = StringField(attr='object_type_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural':
StringField(analyzer='english'),
'suggest': fields.CompletionField(),
})
# To be shown on the detail page
object_type_detail_en = fields.TextField(
attr='object_type_detail_en_indexing', index='no')
material_en = StringField(
attr='material_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
material_detail_en = fields.TextField(attr='material_detail_en_indexing',
index='no')
city_en = StringField(
attr='city_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
country_en = StringField(
attr='country_en_indexing',
analyzer=html_strip_synonyms_en,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='english'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
references_en = fields.TextField(attr='references_en_indexing', index='no')
# To be shown on the detail page
acquired_en = fields.TextField(attr='acquired_en_indexing', index='no')
# To be shown on the detail page
site_found_en = fields.TextField(attr='site_found_en_indexing', index='no')
# To be shown on the detail page
reign_en = fields.TextField(attr='reign_en_indexing', index='no')
# To be shown on the detail page
keywords_en = fields.TextField(attr='keywords_en_indexing', index='no')
# To be shown on the detail page
dynasty_en = fields.TextField(attr='dynasty_en_indexing', index='no')
# New fields
# To be shown on the detail page
credit_line_en = fields.TextField(attr='credit_line_en_indexing',
index='no')
# To be shown on the detail page
region_en = fields.TextField(attr='region_en_indexing', index='no')
# To be shown on the detail page
sub_region_en = fields.TextField(attr='sub_region_en_indexing', index='no')
# To be shown on the detail page
locale_en = fields.TextField(attr='locale_en_indexing', index='no')
# To be shown on the detail page
excavation_en = fields.TextField(attr='excavation_en_indexing', index='no')
# To be shown on the detail page
museum_collection_en = fields.TextField(
attr='museum_collection_en_indexing', index='no')
# To be shown on the detail page
style_en = fields.TextField(attr='style_en_indexing', index='no')
# To be shown on the detail page
culture_en = fields.TextField(attr='culture_en_indexing', index='no')
# To be shown on the detail page
inscriptions_en = fields.TextField(attr='inscriptions_en_indexing',
index='no')
# To be shown on the detail page
provenance_en = fields.TextField(attr='provenance_en_indexing', index='no')
# To be shown on the detail page
exhibitions_en = fields.TextField(attr='exhibitions_en_indexing',
index='no')
# ********************************************************************
# ****************************** Dutch *******************************
# ********************************************************************
title_nl = StringField(
attr='title_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
description_nl = StringField(
attr='description_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
period_nl = StringField(
attr='period_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
period_1_nl = fields.NestedField(
attr='period_1_nl_indexing',
properties={
'name':
StringField(analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
}),
'period_2_nl':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
}),
'period_3_nl':
fields.NestedField(
properties={
'name':
StringField(analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
}),
'period_4_nl':
fields.NestedField(
properties={
'name':
StringField(
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
})
})
})
})
})
primary_object_type_nl = StringField(
attr='primary_object_type_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
'suggest': fields.CompletionField(),
})
object_type_nl = StringField(attr='object_type_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
'suggest': fields.CompletionField(),
})
# To be shown on the detail page
object_type_detail_nl = fields.TextField(
attr='object_type_detail_nl_indexing', index='no')
material_nl = StringField(
attr='material_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
material_detail_nl = fields.TextField(attr='material_detail_nl_indexing',
index='no')
city_nl = StringField(
attr='city_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
country_nl = StringField(
attr='country_nl_indexing',
analyzer=html_strip_synonyms_nl,
fields={
'raw': KeywordField(),
'natural': StringField(analyzer='dutch'),
# 'suggest': fields.CompletionField(),
})
# To be shown on the detail page
keywords_nl = fields.TextField(attr='keywords_nl_indexing', index='no')
# To be shown on the detail page
acquired_nl = fields.TextField(attr='acquired_nl_indexing', index='no')
# To be shown on the detail page
site_found_nl = fields.TextField(attr='site_found_nl_indexing', index='no')
# To be shown on the detail page
reign_nl = fields.TextField(attr='reign_nl_indexing', index='no')
# To be shown on the detail page
references_nl = fields.TextField(attr='references_nl_indexing', index='no')
# To be shown on the detail page
dynasty_nl = fields.TextField(attr='dynasty_nl_indexing', index='no')
# New fields
# To be shown on the detail page
credit_line_nl = fields.TextField(attr='credit_line_nl_indexing',
index='no')
# To be shown on the detail page
region_nl = fields.TextField(attr='region_nl_indexing', index='no')
# To be shown on the detail page
sub_region_nl = fields.TextField(attr='sub_region_nl_indexing', index='no')
# To be shown on the detail page
locale_nl = fields.TextField(attr='locale_nl_indexing', index='no')
# To be shown on the detail page
excavation_nl = fields.TextField(attr='excavation_nl_indexing', index='no')
# To be shown on the detail page
museum_collection_nl = fields.TextField(
attr='museum_collection_nl_indexing', index='no')
# To be shown on the detail page
style_nl = fields.TextField(attr='style_nl_indexing', index='no')
# To be shown on the detail page
culture_nl = fields.TextField(attr='culture_nl_indexing', index='no')
# To be shown on the detail page
inscriptions_nl = fields.TextField(attr='inscriptions_nl_indexing',
index='no')
# To be shown on the detail page
provenance_nl = fields.TextField(attr='provenance_nl_indexing', index='no')
# To be shown on the detail page
exhibitions_nl = fields.TextField(attr='exhibitions_nl_indexing',
index='no')
# ********************************************************************
# ************************** Language independent ********************
# ********************************************************************
dimensions = StringField(
attr='dimensions_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(),
# 'suggest': fields.CompletionField(),
})
object_date_begin = StringField(
attr='object_date_begin_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(),
# 'suggest': fields.CompletionField(),
})
object_date_end = StringField(
attr='object_date_end_indexing',
analyzer=html_strip,
fields={
'raw': KeywordField(),
'natural': StringField(),
# 'suggest': fields.CompletionField(),
})
location = fields.GeoPointField(attr='geo_location_indexing')
# List of 32x32 PNG versions of the images. Full path to.
images = fields.ListField(StringField(attr='images_indexing'))
# List of image URLs.
images_urls = fields.ListField(
fields.ObjectField(attr='images_urls_indexing',
properties={
'th': KeywordField(index="not_analyzed"),
'lr': KeywordField(index="not_analyzed"),
}))
# Classified as by our AI
classified_as = fields.ListField(
StringField(attr='classified_as_indexing',
fields={
'raw': KeywordField(),
}))
# Classified as 1st element
classified_as_1 = StringField(attr='classified_as_1_indexing',
fields={
'raw': KeywordField(),
})
# Classified as 2nd element
classified_as_2 = StringField(attr='classified_as_2_indexing',
fields={
'raw': KeywordField(),
})
# Classified as 3rd element
classified_as_3 = StringField(attr='classified_as_3_indexing',
fields={
'raw': KeywordField(),
})
# ********************************************************************
# ************** Nested fields for search and filtering **************
# ********************************************************************
# # City object
# country = fields.NestedField(
# properties={
# 'name': StringField(
# analyzer=html_strip,
# fields={
# 'raw': KeywordField(),
# 'suggest': fields.CompletionField(),
# }
# ),
# 'info': StringField(analyzer=html_strip),
# 'location': fields.GeoPointField(attr='location_field_indexing'),
# }
# )
#
# location = fields.GeoPointField(attr='location_field_indexing')
class Meta(object):
"""Meta options."""
model = Item # The model associate with this DocType
def get_queryset(self):
"""Filter out items that are not eligible for indexing."""
qs = super(CollectionItemDocument, self).get_queryset()
# qs = qs.select_related('period_node').prefetch_related('images')
filters = []
for field in ['title']:
for language in ['en', 'nl']:
filters.extend([
Q(**{"{}_{}__isnull".format(field, language): True}),
Q(**{"{}_{}__exact".format(field, language): ''}),
])
if filters:
qs = qs.exclude(six.moves.reduce(operator.or_, filters))
# We concatenate ``object_type`` and ``classification`` fields, after
# cleaning them. Therefore, db-only checks don't work here.
ids = []
for item in qs:
if not (item.object_type_en_indexing
and item.object_type_nl_indexing):
ids.append(item.pk)
return qs.exclude(id__in=ids)
def prepare_department(self, instance):
"""Prepare department."""
return instance.department_indexing \
if instance.department_indexing\
else VALUE_NOT_SPECIFIED
def prepare_object_date_begin(self, instance):
"""Prepare material."""
return instance.object_date_begin_indexing
def prepare_object_date_end(self, instance):
"""Prepare material."""
return instance.object_date_end_indexing
# ********************************************************************
# ***************************** English ******************************
# ********************************************************************
def prepare_material_en(self, instance):
"""Prepare material."""
return instance.material_en_indexing \
if instance.material_en_indexing\
else VALUE_NOT_SPECIFIED
def prepare_period_en(self, instance):
"""Prepare state."""
return instance.period_en_indexing \
if instance.period_en_indexing \
else VALUE_NOT_SPECIFIED
def prepare_dynasty_en(self, instance):
"""Prepare dynasty."""
return instance.dynasty_en_indexing \
if instance.dynasty_en_indexing \
else VALUE_NOT_SPECIFIED
def prepare_description_en(self, instance):
"""Prepare description."""
return instance.description_en_indexing \
if instance.description_en_indexing\
else VALUE_NOT_SPECIFIED
def prepare_city_en(self, instance):
"""Prepare city."""
return instance.city_en_indexing \
if instance.city_en_indexing\
else VALUE_NOT_SPECIFIED
def prepare_country_en(self, instance):
"""Prepare country."""
return instance.country_en_indexing \
if instance.country_en_indexing \
else VALUE_NOT_SPECIFIED
# ********************************************************************
# ****************************** Dutch *******************************
# ********************************************************************
def prepare_material_nl(self, instance):
"""Prepare material."""
return instance.material_nl_indexing \
if instance.material_nl_indexing\
else VALUE_NOT_SPECIFIED
def prepare_period_nl(self, instance):
"""Prepare state."""
return instance.period_nl_indexing \
if instance.period_nl_indexing \
else VALUE_NOT_SPECIFIED
def prepare_dynasty_nl(self, instance):
"""Prepare dynasty."""
return instance.dynasty_nl_indexing \
if instance.dynasty_nl_indexing \
else VALUE_NOT_SPECIFIED
def prepare_description_nl(self, instance):
"""Prepare description."""
return instance.description_nl_indexing \
if instance.description_nl_indexing\
else VALUE_NOT_SPECIFIED
def prepare_city_nl(self, instance):
"""Prepare city."""
return instance.city_nl_indexing \
if instance.city_nl_indexing\
else VALUE_NOT_SPECIFIED
def prepare_country_nl(self, instance):
"""Prepare country."""
return instance.country_nl_indexing \
if instance.country_nl_indexing \
else VALUE_NOT_SPECIFIED
class EFOTraitDocument(Document):
"""EFOTrait elasticsearch document"""
id = fields.TextField()
label = fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}
)
description = fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}
)
synonyms = fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
)
mapped_terms = fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
)
url = fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword')
}
)
traitcategory = fields.ObjectField(
properties={
'label': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}
),
'parent': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
)
}
)
scores_direct_associations = fields.ObjectField(
properties={
'id': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}
),
'name': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
),
'trait_reported': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
)
}
)
scores_child_associations = fields.ObjectField(
properties={
'id': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}
),
'name': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
),
'trait_reported': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
)
}
)
parent_traits = fields.ObjectField(
properties={
'id': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}
),
'label': fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}
)
}
)
class Django(object):
"""Inner nested class Django."""
model = EFOTrait_Ontology # The model associated with this Document
related_models = [TraitCategory]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, TraitCategory):
return related_instance.efotraits_ontology.all()
class PublicationDocument(Document):
"""Publication elasticsearch document"""
id = fields.TextField()
title = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
journal = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
pub_year = fields.TextField(analyzer=html_strip,
fields={
'raw':
fields.TextField(analyzer='keyword'),
})
PMID = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
firstauthor = fields.TextField(analyzer=html_strip,
fields={
'raw':
fields.TextField(analyzer='keyword'),
})
authors = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
doi = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
scores_count = fields.IntegerField()
publication_score = fields.ObjectField(
properties={
'id':
fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}),
'name':
fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}),
'trait_reported':
fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
}),
'trait_efo':
fields.ObjectField(
properties={
'id':
fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
}),
'label':
fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
'suggest': fields.CompletionField()
})
})
})
class Django(object):
"""Inner nested class Django."""
model = Publication # The model associated with this Document
class AddressDocument(DocType):
"""Address Elasticsearch document."""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
street = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
)
house_number = StringField(analyzer=html_strip)
appendix = StringField(analyzer=html_strip)
zip_code = StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
)
# ********************************************************************
# ********** Additional fields for search and filtering **************
# ********************************************************************
# City object
city = fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'info': StringField(analyzer=html_strip),
'location': fields.GeoPointField(attr='location_field_indexing'),
'country': fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'info': StringField(analyzer=html_strip),
'location': fields.GeoPointField(
attr='location_field_indexing'
)
}
)
}
)
# Country object
country = fields.NestedField(
attr='country_indexing',
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'city': fields.ObjectField(
properties={
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
},
),
},
),
},
)
# Continent object
continent = fields.NestedField(
attr='continent_indexing',
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}
),
'country': fields.NestedField(
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
),
'city': fields.NestedField(
properties={
'id': fields.IntegerField(),
'name': StringField(
analyzer=html_strip,
fields={
'raw': KeywordField(),
}
)
}
)
}
)
}
)
location = fields.GeoPointField(attr='location_field_indexing')
class Meta(object):
"""Meta options."""
model = Address # The model associate with this DocType
class DatasetsDoc(DocType):
id = fields.IntegerField()
slug = fields.KeywordField()
title = fields.TextField(analyzer=polish_analyzer,
fields={
'raw': fields.KeywordField(),
'suggest': fields.CompletionField()
})
version = fields.KeywordField()
url = fields.KeywordField()
notes = fields.TextField(analyzer=polish_analyzer,
fields={
'raw': fields.KeywordField(),
})
institution = fields.NestedField(
attr='organization',
properties={
'id':
fields.IntegerField(),
'title':
fields.TextField(analyzer=polish_analyzer,
fields={'raw': fields.KeywordField()})
})
category = fields.NestedField(
properties={
'id':
fields.IntegerField(),
'title':
fields.TextField(analyzer=polish_analyzer,
fields={'raw': fields.KeywordField()})
})
resources = fields.NestedField(
properties={
'id':
fields.IntegerField(),
'title':
fields.TextField(analyzer=polish_analyzer,
fields={'raw': fields.KeywordField()})
})
applications = fields.NestedField(
properties={
'id':
fields.IntegerField(),
'title':
fields.TextField(analyzer=polish_analyzer,
fields={'raw': fields.KeywordField()})
})
articles = fields.NestedField(
properties={
'id':
fields.IntegerField(),
'title':
fields.TextField(analyzer=polish_analyzer,
fields={'raw': fields.KeywordField()})
})
tags = fields.KeywordField(attr='tags_list', multi=True)
# customfields = fields.TextField()
formats = fields.KeywordField(attr='formats', multi=True)
license_condition_db_or_copyrighted = fields.TextField()
license_condition_modification = fields.BooleanField()
license_condition_original = fields.BooleanField()
license_condition_responsibilities = fields.TextField()
license_condition_source = fields.BooleanField()
license_condition_timestamp = fields.BooleanField()
license_name = fields.StringField(attr='license_name')
license_description = fields.StringField(attr='license_description')
update_frequency = fields.KeywordField()
openness_scores = fields.IntegerField(attr='openness_scores', multi=True)
users_following = fields.KeywordField(attr='users_following_list',
multi=True)
views_count = fields.IntegerField()
downloads_count = fields.IntegerField()
status = fields.KeywordField()
modified = fields.DateField()
last_modified_resource = fields.DateField(attr='last_modified_resource')
created = fields.DateField()
class Meta:
doc_type = 'dataset'
model = Dataset
related_models = [
Organization, Category, Application, Article, Resource,
UserFollowingDataset
]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, UserFollowingDataset):
return related_instance.follower.followed_applications.all()
if isinstance(related_instance, Application):
return related_instance.datasets.all()
if isinstance(related_instance, Resource):
return related_instance.dataset
def get_queryset(self):
return self._doc_type.model.objects.filter(status='published')
class BaseCourseDocument(OrganizationsMixin, BaseDocument):
"""
Base course document index.
Contains common fields and logic for Course and CourseRun indexes.
"""
authoring_organizations = fields.TextField(
multi=True,
fields={
'suggest':
fields.CompletionField(),
'edge_ngram_completion':
fields.TextField(analyzer=edge_ngram_completion),
'raw':
fields.KeywordField(),
'lower':
fields.TextField(analyzer=case_insensitive_keyword)
},
)
authoring_organization_bodies = fields.TextField(multi=True)
key = fields.TextField(fields={'raw': fields.KeywordField()})
title = fields.TextField(
analyzer=synonym_text,
fields={
'suggest':
fields.CompletionField(),
'edge_ngram_completion':
fields.TextField(analyzer=edge_ngram_completion),
},
)
first_enrollable_paid_seat_price = fields.IntegerField()
full_description = fields.TextField(analyzer=html_strip)
image_url = fields.TextField()
logo_image_urls = fields.TextField(multi=True)
level_type = fields.TextField(
fields={
'raw': fields.KeywordField(),
'lower': fields.TextField(analyzer=case_insensitive_keyword)
})
partner = fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.KeywordField(),
'lower': fields.TextField(analyzer=case_insensitive_keyword)
})
outcome = fields.TextField()
org = fields.TextField(
analyzer=html_strip,
fields={
'raw': fields.KeywordField(),
'lower': fields.TextField(analyzer=case_insensitive_keyword)
})
subject_uuids = fields.KeywordField(multi=True)
short_description = fields.TextField(analyzer=html_strip)
seat_types = fields.KeywordField(multi=True)
subjects = fields.TextField(
analyzer=html_strip,
fields={'raw': fields.KeywordField(multi=True)},
multi=True)
sponsoring_organizations = fields.TextField(multi=True)
def prepare_first_enrollable_paid_seat_price(self, obj):
return obj.first_enrollable_paid_seat_price
def prepare_level_type(self, obj):
return obj.level_type.name if obj.level_type else None
def prepare_logo_image_urls(self, obj):
orgs = obj.authoring_organizations.all()
return [org.logo_image.url for org in orgs if org.logo_image]
def prepare_organizations(self, obj):
return list(
set(
self.prepare_authoring_organizations(obj) +
self.prepare_sponsoring_organizations(obj)))
def prepare_subjects(self, obj):
return [subject.name for subject in obj.subjects.all()]
def prepare_subject_uuids(self, obj):
return [str(subject.uuid) for subject in obj.subjects.all()]
def prepare_sponsoring_organizations(self, obj):
return self._prepare_organizations(obj.sponsoring_organizations.all())
class ProgramDocument(BaseDocument, OrganizationsMixin):
"""
Program Elasticsearch document.
"""
authoring_organization_uuids = fields.KeywordField(multi=True)
authoring_organizations = fields.TextField(
multi=True,
fields={
'suggest': fields.CompletionField(),
'edge_ngram_completion': fields.TextField(analyzer=edge_ngram_completion),
'raw': fields.KeywordField(),
'lower': fields.TextField(analyzer=case_insensitive_keyword)
},
)
authoring_organization_bodies = fields.TextField(multi=True)
credit_backing_organizations = fields.TextField(multi=True)
card_image_url = fields.TextField()
hidden = fields.BooleanField()
is_program_eligible_for_one_click_purchase = fields.BooleanField()
language = fields.TextField(multi=True)
marketing_url = fields.TextField()
min_hours_effort_per_week = fields.IntegerField()
max_hours_effort_per_week = fields.IntegerField()
partner = fields.TextField(
analyzer=html_strip,
fields={'raw': fields.KeywordField(), 'lower': fields.TextField(analyzer=case_insensitive_keyword)}
)
published = fields.BooleanField()
subtitle = fields.TextField(analyzer=html_strip)
status = fields.KeywordField()
search_card_display = fields.TextField(multi=True)
subject_uuids = fields.KeywordField(multi=True)
staff_uuids = fields.KeywordField(multi=True)
start = fields.DateField()
seat_types = fields.KeywordField(multi=True)
title = fields.TextField(
analyzer=synonym_text,
fields={
'suggest': fields.CompletionField(),
'edge_ngram_completion': fields.TextField(analyzer=edge_ngram_completion),
},
)
type = fields.TextField(
analyzer=html_strip,
fields={'raw': fields.KeywordField(), 'lower': fields.TextField(analyzer=case_insensitive_keyword)}
)
weeks_to_complete_min = fields.IntegerField()
weeks_to_complete_max = fields.IntegerField()
def prepare_aggregation_key(self, obj):
return 'program:{}'.format(obj.uuid)
def prepare_credit_backing_organizations(self, obj):
return self._prepare_organizations(obj.credit_backing_organizations.all())
def prepare_language(self, obj):
return [self._prepare_language(language) for language in obj.languages]
def prepare_organizations(self, obj):
return self.prepare_authoring_organizations(obj) + self.prepare_credit_backing_organizations(obj)
def prepare_partner(self, obj):
return obj.partner.short_code
def prepare_published(self, obj):
return obj.status == ProgramStatus.Active
def prepare_seat_types(self, obj):
return [seat_type.slug for seat_type in obj.seat_types]
def prepare_search_card_display(self, obj):
try:
degree = Degree.objects.get(uuid=obj.uuid)
except Degree.DoesNotExist:
return []
return [degree.search_card_ranking, degree.search_card_cost, degree.search_card_courses]
def prepare_subject_uuids(self, obj):
return [str(subject.uuid) for subject in obj.subjects]
def prepare_staff_uuids(self, obj):
return list({str(staff.uuid) for course_run in obj.course_runs for staff in course_run.staff.all()})
def prepare_type(self, obj):
return obj.type.name_t
def get_queryset(self):
return super().get_queryset().select_related('type').select_related('partner')
class Django:
"""
Django Elasticsearch DSL ORM Meta.
"""
model = Program
class Meta:
"""
Meta options.
"""
parallel_indexing = True
queryset_pagination = settings.ELASTICSEARCH_DSL_QUERYSET_PAGINATION
