def get_index(name):
index_name = '%s_%s' % (
settings.ELASTICSEARCH_INDEX_PREFIX,
name
)
# if settings.ELASTICSEARCH_INDEX_PREFIX == 'froide_test':
# index_name += '_%s' % threading.get_ident()
index = Index(index_name)
# See Elasticsearch Indices API reference for available settings
index.settings(
number_of_shards=1,
number_of_replicas=0
)
return index
from django_elasticsearch_dsl import DocType, Index, fields
from elasticsearch_dsl import Completion
from .models import Package, Product
packages = Index('packages')
packages.settings(
number_of_shards=4,
number_of_replicas=0
)
products = Index('products')
products.settings(
number_of_shards=4,
number_of_replicas=0
)
@packages.doc_type
class PackageDocument(DocType):
# manager = fields.TextField(attr='_get_manager_product')
package_name = fields.TextField(attr='_get_package_name',
fields={
'raw': fields.StringField(analyzer='keyword'),
'suggest': fields.CompletionField(attr='_get_package_name')
# contexts=[{"name": "manager", "type": "category", "path": "manager"}]),
}
)
class Meta:
model = Package
fields = ['name']
from django.apps import apps
from django_elasticsearch_dsl import DocType, Index, fields
from mcod import settings
from mcod.datasets.documents import datasets_field
from mcod.lib.search.analyzers import polish_analyzer, standard_analyzer
Organization = apps.get_model('organizations', 'Organization')
Dataset = apps.get_model('datasets', 'Dataset')
INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES['institutions'])
INDEX.settings(**settings.ELASTICSEARCH_DEFAULT_INDEX_SETTINGS)
@INDEX.doc_type
class InstitutionDoc(DocType):
id = fields.IntegerField()
slug = fields.TextField()
title = fields.TextField(analyzer=polish_analyzer,
fields={
'raw': fields.KeywordField(),
'suggest': fields.CompletionField()
})
description = fields.TextField(analyzer=polish_analyzer,
fields={
'raw': fields.KeywordField(),
})
image_url = fields.TextField(attr='image_url')
postal_code = fields.KeywordField()
city = fields.TextField(analyzer=standard_analyzer,
from elasticsearch_dsl import analyzer
from django_elasticsearch_dsl import Index
note_index = Index('note')
note_index.settings(number_of_shards=1, number_of_replicas=1)
html_strip = analyzer('html_strip',
tokenizer="standard",
filter=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"])
from django_elasticsearch_dsl import DocType, Index, fields
from . import models
entries = Index('docket-entries')
entries.settings(number_of_shards=1, number_of_replicas=0)
# @entries.doc_type
class DocketEntryDocument(DocType):
court = fields.ObjectField(properties={
'description': fields.TextField(),
'name': fields.KeywordField(),
})
case = fields.ObjectField(
properties={
'year': fields.KeywordField(),
'number': fields.KeywordField(),
'office': fields.KeywordField(),
'type': fields.KeywordField(),
})
class Meta:
model = models.DocketEntry
fields = [
'case_number',
'case_name',
'title',
'time_filed',
]
from django_elasticsearch_dsl import DocType, Index, fields
from elasticsearch_dsl import analyzer, token_filter
from ..order.models import Order
from ..product.models import Product
from ..userprofile.models import User
storefront = Index('storefront')
storefront.settings(number_of_shards=1, number_of_replicas=0)
partial_words = token_filter('partial_words',
'edge_ngram',
min_gram=3,
max_gram=15)
title_analyzer = analyzer('title_analyzer',
tokenizer='standard',
filter=[partial_words, 'lowercase'])
email_analyzer = analyzer('email_analyzer', tokenizer='uax_url_email')
@storefront.doc_type
class ProductDocument(DocType):
title = fields.StringField(analyzer=title_analyzer)
def prepare_title(self, instance):
return instance.name
class Meta:
model = Product
fields = ['name', 'description', 'is_published']
from elasticsearch_dsl import analyzer
from django_elasticsearch_dsl import DocType, Index, fields
from .models import Car, Manufacturer, Ad
car = Index('test_cars')
car.settings(
number_of_shards=1,
number_of_replicas=0
)
html_strip = analyzer(
'html_strip',
tokenizer="standard",
filter=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"]
)
@car.doc_type
class CarDocument(DocType):
manufacturer = fields.ObjectField(properties={
'name': fields.StringField(),
'country': fields.StringField(),
})
ads = fields.NestedField(properties={
'description': fields.StringField(analyzer=html_strip),
'title': fields.StringField(),
from django_elasticsearch_dsl import Document, Index, fields
from packages.models import Package
package_index = Index("packages")
package_index.settings(number_of_shards=1, number_of_replicas=0)
@package_index.document
class PackageDocument(Document):
class Django:
model = Package
fields = [
"id",
"author",
"author_email",
"bugtrack_url",
"classifiers",
"description",
"description_content_type",
"docs_url",
"download_url",
"downloads",
"home_page",
"keywords",
"license",
"maintainer",
"maintainer_email",
"name",
"package_url",
from django.conf import settings
from django.db.models import QuerySet
from django_elasticsearch_dsl import Index
from metarecord.models import Classification
from search_indices import get_finnish_analyzer
from search_indices.documents.base import BaseDocument
# Name of the Elasticsearch index
INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__])
finnish_analyzer = get_finnish_analyzer()
INDEX.analyzer(finnish_analyzer)
INDEX.settings(max_result_window=500000, )
@INDEX.document
class ClassificationDocument(BaseDocument):
class Django:
model = Classification
def get_queryset(self) -> QuerySet:
return Classification.objects.latest_version()
if hasattr(settings, 'ELASTIC_PREFIX') and settings.ELASTIC_PREFIX:
people_index_name = settings.ELASTIC_PREFIX + '_' + people_index_name
# Startup search document
startup_index_name = 'startup'
if hasattr(settings, 'ELASTIC_PREFIX') and settings.ELASTIC_PREFIX:
startup_index_name = settings.ELASTIC_PREFIX + '_' + startup_index_name
# Job search document
job_index_name = 'job'
if hasattr(settings, 'ELASTIC_PREFIX') and settings.ELASTIC_PREFIX:
job_index_name = settings.ELASTIC_PREFIX + '_' + job_index_name
people = Index(people_index_name)
people.settings(
number_of_shards=1,
number_of_replicas=0,
)
startup = Index(startup_index_name)
startup.settings(
number_of_shards=1,
number_of_replicas=0,
)
job = Index(job_index_name)
job.settings(
number_of_shards=1,
number_of_replicas=0,
)
leave_default = analyzer('leave_default',
from elasticsearch_dsl import analyzer
from django_elasticsearch_dsl import DocType, Index, fields
from elasticsearch_dsl import connections
from .models import Measurement, Device
connections.create_connection(alias='default', http_auth=(
'elastic', 'hR6DNpugI77mueYKNBEulFH3'), hosts=["https://d284504808e347ccb2779337f23759e2.ap-southeast-2.aws.found.io:9243"])
measurement_index = Index('measurements')
measurement_index.settings(
number_of_shards=1,
number_of_replicas=0
)
@measurement_index.doc_type
class MeasurementDocument(DocType):
"""Article elasticsearch document"""
id = fields.LongField(attr='id')
key = fields.StringField()
value = fields.StringField()
location = fields.StringField()
device = fields.StringField(attr='device_id')
time = fields.IntegerField()
class Meta:
model = Measurement
from django_elasticsearch_dsl import DocType, Index, fields
from elasticsearch_dsl import Completion
from .models import Contact, ContactGroup
contacts = Index('contacts')
contacts.settings(number_of_shards=4, number_of_replicas=0)
@contacts.doc_type
class ContactDocument(DocType):
full_name = fields.TextField(
attr='_get_fullname',
fields={
'raw': fields.StringField(analyzer='keyword'),
'suggest': fields.CompletionField(attr='_get_fullname')
})
class Meta:
model = Contact
fields = ['first_name', 'last_name']
from elasticsearch_dsl import analyzer
from django_elasticsearch_dsl import DocType, Index, fields
from pokedex import models as pokemon_models
pokemon_index = Index('pokemon')
pokemon_index.settings(number_of_shards=1, number_of_replicas=0)
html_strip = analyzer('html_strip',
tokenizer="standard",
filter=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"])
# @pokemon_index.doc_type
# class PokemonDocument(DocType):
# class Meta:
# model = pokemon_models.Pokemon
# fields = [
# 'id',
# 'name',
# 'type1',
# 'type2',
# 'total_stats',
# 'generation',
# 'is_legendary',
# 'created',
# ]
from django_elasticsearch_dsl import DocType, Index
from upstarter.models import Project
project_index = Index('projects')
project_index.settings(
number_of_shards=1,
number_of_replicas=0,
)
@project_index.doc_type
class ProjectDocument(DocType):
class Meta:
model = Project
fields = ('name', 'description', 'tags')
from django_elasticsearch_dsl import DocType, Index, fields
from django_elasticsearch_dsl_drf.compat import KeywordField, StringField
from books.models import City
from .analyzers import html_strip
__all__ = ('CityDocument',)
INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__])
# See Elasticsearch Indices API reference for available settings
INDEX.settings(
number_of_shards=1,
number_of_replicas=1,
blocks={'read_only_allow_delete': False},
# read_only_allow_delete=False
)
@INDEX.doc_type
class CityDocument(DocType):
"""City Elasticsearch document.
This document has been created purely for testing out complex fields.
"""
# In different parts of the code different fields are used. There are
# a couple of use cases: (1) more-like-this functionality, where `title`,
# `description` and `summary` fields are used, (2) search and filtering
# functionality where all of the fields are used.
from elasticsearch_dsl import analyzer
from django_elasticsearch_dsl import DocType, Index, fields
from .models import Car, Manufacturer, Ad
car_index = Index('test_cars')
car_index.settings(number_of_shards=1, number_of_replicas=0)
html_strip = analyzer('html_strip',
tokenizer="standard",
filter=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"])
@car_index.doc_type
class CarDocument(DocType):
manufacturer = fields.ObjectField(properties={
'name': fields.StringField(),
'country': fields.StringField(),
})
ads = fields.NestedField(
properties={
'description': fields.StringField(analyzer=html_strip),
'title': fields.StringField(),
'pk': fields.IntegerField(),
})
class Meta:
model = Car
fields = [
from django_elasticsearch_dsl import DocType, Index, fields
from elasticsearch_dsl import analyzer
from .models import *
public = Index('publics')
public.settings(
number_of_shards=1,
number_of_replicas=0
)
html_strip = analyzer(
'html_strip',
tokenizer="standard",
filter=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"]
)
@public.doc_type
class PublicDocument(DocType):
context = fields.TextField(attr="ctx")
section = fields.ObjectField(properties={
'name': fields.StringField(analyzer=html_strip),
})
# section = fields.ObjectField(properties={
# 'to_dict': fields.ObjectField()
# })
author = fields.ObjectField(properties={
'nickname': fields.StringField(analyzer=html_strip),
})
topic_public = fields.ObjectField(properties={
from elasticsearch_dsl import analyzer
from django_elasticsearch_dsl import DocType, Index, fields
from article import models as articles_models
article_index = Index('articles')
article_index.settings(number_of_shards=1, number_of_replicas=0)
html_strip = analyzer('html_strip',
tokenizer="standard",
filter=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"])
@article_index.doc_type
class ArticleDocument(DocType):
"""Article elasticsearch document"""
id = fields.IntegerField(attr='id')
title = fields.StringField(analyzer=html_strip,
fields={
'raw':
fields.StringField(analyzer='keyword'),
})
body = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
author = fields.IntegerField(attr='author_id')
created = fields.DateField()
from django_elasticsearch_dsl import DocType, Index
from .models import Manga
# Name of the Elasticsearch index
manga = Index("mangas")
# See Elasticsearch Indices API reference for available settings
manga.settings(number_of_shards=1, number_of_replicas=0)
@manga.doc_type
class MangaDocument(DocType):
class Meta:
model = Manga # The model associated with this DocType
# The fields of the model you want to be indexed in Elasticsearch
fields = [
"name",
"web_source",
]
from django_elasticsearch_dsl import Document, Index, fields
from elasticsearch_dsl import analyzer
from elasticsearch_dsl.analysis import token_filter
from data_refinery_common.models.experiment import Experiment
experiment_index = Index("experiments")
experiment_index.settings(number_of_shards=1, number_of_replicas=0, max_result_window=9999999)
# via https://django-elasticsearch-dsl-drf.readthedocs.io/en/0.17.2/advanced_usage_examples.html?highlight=ngram#id8
# via https://github.com/barseghyanartur/django-elasticsearch-dsl-drf/issues/110
edge_ngram_completion_filter = token_filter(
"edge_ngram_completion_filter", type="edge_ngram", min_gram=3, max_gram=12
)
html_strip = analyzer(
"html_strip",
tokenizer="whitespace",
filter=[edge_ngram_completion_filter, "standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"],
)
html_strip_no_ngram = analyzer(
"html_strip_no_ngram",
tokenizer="standard",
filter=["standard", "lowercase", "stop"],
char_filter=["html_strip"],
)
html_strip_no_stop = analyzer(
"html_strip_no_stop",
tokenizer="whitespace",
filter=["standard", "lowercase"],
char_filter=["html_strip"],
from django_elasticsearch_dsl import (
Document,
fields,
Index,
)
from .models import ElasticDemo
PUBLISHER_INDEX = Index('elastic_demo')
PUBLISHER_INDEX.settings(number_of_shards=1, number_of_replicas=1)
@PUBLISHER_INDEX.doc_type
class NewsDocument(Document):
id = fields.IntegerField(attr='id')
fielddata = True
title = fields.TextField(fields={'raw': {
'type': 'keyword',
}})
content = fields.TextField(fields={'raw': {
'type': 'keyword',
}}, )
class Django(object):
model = ElasticDemo
# Name of the Elasticsearch index
from django.conf import settings
from django_elasticsearch_dsl import Index, DEDField, Integer
from elasticsearch_dsl import analyzer, token_filter
class RelatedToValueList(DEDField, Integer):
def get_value_from_instance(self, data):
return [obj.id for obj in super().get_value_from_instance(data)]
mainIndex = Index(settings.ELASTICSEARCH_INDEX)
# See Elasticsearch Indices API reference for available settings
mainIndex.settings(number_of_shards=1, number_of_replicas=0)
autocomplete_filter = token_filter(
"autocomplete_filter",
"edge_ngram",
min_gram=1,
max_gram=20,
)
# Using this analyzer with an empty field fails, so we're using methods instead that add a space
autocomplete_analyzer = analyzer(
'autocomplete',
tokenizer="standard",
filter=["lowercase", autocomplete_filter],
)
mainIndex.analyzer(autocomplete_analyzer)
from elasticsearch_dsl import analyzer
from django_elasticsearch_dsl import Document, Index, fields
from spotify import models as spotify_models
spotify_index = Index('spotify')
spotify_index.settings(
number_of_shards=5,
number_of_replicas=2,
)
html_strip = analyzer('html_strip',
tokenizer="standard",
filter=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"])
@spotify_index.doc_type
class SpotifyDocument(Document):
"""Spotify elasticsearch document"""
acousticness = fields.FloatField()
artists = fields.KeywordField(multi=True,
analyzer=html_strip,
fields={
'raw':
fields.KeywordField(analyzer='keyword'),
})
danceability = fields.FloatField()
duration_ms = fields.IntegerField()
energy = fields.FloatField()
from django.conf import settings
from django_elasticsearch_dsl import Document, Index, fields
from elasticsearch_dsl import analyzer
from books.models import Book
# Name of the Elasticsearch index
INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__])
# See Elasticsearch Indices API reference for available settings
INDEX.settings(
number_of_shards=1,
number_of_replicas=1,
)
@INDEX.doc_type
class BookDocument(Document):
"""Book Elasticsearch document."""
id = fields.IntegerField(attr='id')
title = fields.TextField(fields={
'raw': fields.KeywordField(),
})
description = fields.TextField(fields={
'raw': fields.KeywordField(),
})
summary = fields.TextField(fields={
'raw': fields.KeywordField(),
})
from django_elasticsearch_dsl import DocType, Index, fields
from elasticsearch_dsl import analyzer, token_filter
from ..account.models import User
from ..order.models import Order
from ..product.models import Product
storefront = Index('storefront')
storefront.settings(number_of_shards=1, number_of_replicas=0)
partial_words = token_filter(
'partial_words', 'edge_ngram', min_gram=3, max_gram=15)
title_analyzer = analyzer(
'title_analyzer',
tokenizer='standard',
filter=[partial_words, 'lowercase'])
email_analyzer = analyzer('email_analyzer', tokenizer='uax_url_email')
@storefront.doc_type
class ProductDocument(DocType):
title = fields.StringField(analyzer=title_analyzer)
def prepare_title(self, instance):
return instance.name
class Meta:
model = Product
fields = ['name', 'description', 'is_published']
from django_elasticsearch_dsl import (
DocType,
fields,
Index,
)
from ..incidents.models import Incident
incident_index = Index('incident')
incident_index.settings(number_of_shards=1, number_of_replicas=0)
@incident_index.doc_type
class IncidentDocument(DocType):
title = fields.StringField(attr='incident_title',
fields={
'raw':
fields.StringField(analyzer='keyword'),
'suggest': fields.Completion(),
})
description = fields.StringField(
attr='description',
fields={
'raw': fields.StringField(analyzer='keyword'),
})
class Meta:
model = Incident
from datetime import datetime
from django_elasticsearch_dsl import DocType, Index, fields
from ..models import Device, User
from elasticsearch import TransportError
from elasticsearch_dsl import GeoPoint
# Name of the Elasticsearch index
INDEX = 'device'
device = Index(INDEX)
# See Elasticsearch Indices API reference for available settings
device.settings(number_of_shards=1, number_of_replicas=0)
@device.doc_type
class DeviceDocument(DocType):
date_modified = fields.DateField()
user = fields.ObjectField(properties={
'mobile': fields.StringField(),
'email': fields.StringField(),
})
location = fields.GeoPointField()
tag_suggest = fields.CompletionField()
stores = fields.NestedField(properties={
'store_id': fields.StringField(),
'date': fields.DateField(),
from django_elasticsearch_dsl import DocType, Index, fields
from mcod.users.models import UserFollowingDataset
from mcod import settings
from mcod.lib.search.fields import TranslatedTextField, TranslatedKeywordField, TranslatedKeywordsList
Dataset = apps.get_model('datasets', 'Dataset')
Organization = apps.get_model('organizations', 'Organization')
Category = apps.get_model('categories', 'Category')
Tag = apps.get_model('tags', 'Tag')
Resource = apps.get_model('resources', 'Resource')
Article = apps.get_model('articles', 'Article')
Application = apps.get_model('applications', 'Application')
INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES['datasets'])
INDEX.settings(**settings.ELASTICSEARCH_DSL_INDEX_SETTINGS)
def datasets_field(**kwargs):
return fields.NestedField(properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title'),
'notes': TranslatedTextField('notes'),
'category': fields.KeywordField(attr='category.title'),
'formats': fields.KeywordField(attr='formats', multi=True),
'downloads_count': fields.IntegerField(attr='downloads_count'),
'views_count': fields.IntegerField(attr='views_count'),
'openness_scores': fields.IntegerField(attr='openness_scores'),
'modified': fields.DateField(),
'slug': TranslatedKeywordField('slug'),
'verified': fields.DateField(),
from django.conf import settings
from django_elasticsearch_dsl import Document, Index, fields
from elasticsearch_dsl import analyzer
from products.models import Product
# Name of the Elasticsearch index
PRODUCT_INDEX = Index('product')
# See Elasticsearch Indices API reference for available settings
PRODUCT_INDEX.settings(number_of_shards=1, number_of_replicas=1)
html_strip = analyzer('html_strip',
tokenizer="standard",
bool=["standard", "lowercase", "stop", "snowball"],
char_filter=["html_strip"])
@PRODUCT_INDEX.doc_type
class ProductDocument(Document):
"""Products Elasticsearch document."""
id = fields.IntegerField(attr='id')
name = fields.TextField(analyzer=html_strip,
fields={
'raw': fields.TextField(analyzer='keyword'),
})
slug = fields.TextField(analyzer=html_strip,
fields={
import logging
from django.conf import settings
from django_elasticsearch_dsl import Document, Index, fields
from elasticsearch import Elasticsearch
from readthedocs.projects.models import HTMLFile, Project
project_conf = settings.ES_INDEXES['project']
project_index = Index(project_conf['name'])
project_index.settings(**project_conf['settings'])
page_conf = settings.ES_INDEXES['page']
page_index = Index(page_conf['name'])
page_index.settings(**page_conf['settings'])
log = logging.getLogger(__name__)
class RTDDocTypeMixin:
def update(self, *args, **kwargs):
# Hack a fix to our broken connection pooling
# This creates a new connection on every request,
# but actually works :)
log.info('Hacking Elastic indexing to fix connection pooling')
self.using = Elasticsearch(**settings.ELASTICSEARCH_DSL['default'])
super().update(*args, **kwargs)
@project_index.document
class ProjectDocument(RTDDocTypeMixin, Document):
from django_elasticsearch_dsl import DocType, Index, fields
from blog.models import Post, User
# Name of the Elasticsearch index
search_index = Index('blog')
# see Elasticsearch Indices API reference for available settings
search_index.settings(number_of_shards=1, number_of_replicas=0)
@search_index.doc_type
class PostDocument(DocType):
author = fields.NestedField(properties={
'first_name': fields.TextField(),
'last_name': fields.TextField(),
'username': fields.TextField(),
'pk': fields.IntegerField(),
},
include_in_root=True)
class Meta:
model = Post # The model associated with this DocType
# The fields of the you want to be indexed in Elasticsearch
fields = ['title', 'body', 'created']
related_models = [User]
def get_instances_from_related(self, related_instance):
"""If related_models is set, define how to retrieve the Post instance(s) from the related model."""
if isinstance(related_instance, User):
return related_instance.blog_posts.all()
from django_elasticsearch_dsl import DocType, Index, fields
from products.models import Product, ProductCategory
product = Index('products')
product.settings(number_of_shards=1, number_of_replicas=0)
@product.doc_type
class ProductDocument(DocType):
category = fields.NestedField(properties={
'name': fields.TextField(),
})
discount = fields.FloatField(attr='discount')
class Meta:
model = Product
fields = [
'name',
'regular_price',
'final_price',
'is_available',
'description',
'timestamp',
]
related_models = [ProductCategory]
def get_queryset(self):
return super(ProductDocument,
import logging
from django.conf import settings
from django_elasticsearch_dsl import DocType, Index, fields
from readthedocs.projects.models import HTMLFile, Project
from readthedocs.sphinx_domains.models import SphinxDomain
project_conf = settings.ES_INDEXES['project']
project_index = Index(project_conf['name'])
project_index.settings(**project_conf['settings'])
page_conf = settings.ES_INDEXES['page']
page_index = Index(page_conf['name'])
page_index.settings(**page_conf['settings'])
domain_conf = settings.ES_INDEXES['domain']
domain_index = Index(domain_conf['name'])
domain_index.settings(**domain_conf['settings'])
log = logging.getLogger(__name__)
@domain_index.doc_type
class SphinxDomainDocument(DocType):
project = fields.KeywordField(attr='project.slug')
version = fields.KeywordField(attr='version.slug')
role_name = fields.KeywordField(attr='role_name')
# For linking to the URL
