예제 #1
0
def configure_connection():
    """Configure Elasticsearch default connection."""
    connections_default = {
        'hosts': [settings.ES_URL],
        'verify_certs': settings.ES_VERIFY_CERTS,
    }
    connections.configure(default=connections_default)
예제 #2
0
파일: apps.py 프로젝트: honcho-cheng/kuma
    def ready(self):
        super(WikiConfig, self).ready()

        # Configure Elasticsearch connections for connection pooling.
        es_connections.configure(
            default={
                'hosts': settings.ES_URLS,
            },
            indexing={
                'hosts': settings.ES_URLS,
                'timeout': settings.ES_INDEXING_TIMEOUT,
            },
        )

        # connect some signal handlers for the wiki models
        Document = self.get_model('Document')
        signals.post_save.connect(self.on_document_save,
                                  sender=Document,
                                  dispatch_uid='wiki.document.post_save')
        render_done.connect(self.on_render_done,
                            dispatch_uid='wiki.document.render_done')

        Revision = self.get_model('Revision')
        signals.post_save.connect(self.on_revision_save,
                                  sender=Revision,
                                  dispatch_uid='wiki.revision.post_save')

        DocumentZone = self.get_model('DocumentZone')
        signals.post_save.connect(self.on_zone_save,
                                  sender=DocumentZone,
                                  dispatch_uid='wiki.zone.post_save')
예제 #3
0
    def setUpClass(cls):
        if cls._overridden_settings:
            cls._cls_overridden_context = override_settings(
                **cls._overridden_settings)
            cls._cls_overridden_context.enable()

        connections.configure(**settings.ELASTICSEARCH_CONNECTIONS)
        cls.es_client = cls._get_client()

        IngestClient(cls.es_client).put_pipeline(
            id='ingest_attachment',
            body={
                'description':
                "Extract attachment information",
                'processors': [{
                    "attachment": {
                        "field": "data",
                        "indexed_chars": "-1"
                    },
                    "remove": {
                        "field": "data"
                    }
                }]
            })

        super().setUpClass()
예제 #4
0
def configure_connection():
    """Configure Elasticsearch default connection."""
    if settings.ES_USE_AWS_AUTH:
        es_protocol = {
            'http': 80,
            'https': 443,
        }
        es_host = urlparse(settings.ES_URL)
        es_port = es_host.port if es_host.port else es_protocol.get(
            es_host.scheme)
        auth = AWSRequestsAuth(
            aws_access_key=settings.AWS_ELASTICSEARCH_KEY,
            aws_secret_access_key=settings.AWS_ELASTICSEARCH_SECRET,
            aws_host=es_host.netloc,
            aws_region=settings.AWS_ELASTICSEARCH_REGION,
            aws_service='es',
        )
        connections_default = {
            'hosts': [es_host.netloc],
            'port': es_port,
            'use_ssl': settings.ES_USE_SSL,
            'verify_certs': settings.ES_VERIFY_CERTS,
            'http_auth': auth,
            'connection_class': RequestsHttpConnection,
        }
    else:
        connections_default = {
            'hosts': [settings.ES_URL],
            'verify_certs': settings.ES_VERIFY_CERTS,
        }

    connections.configure(default=connections_default, )
예제 #5
0
파일: apps.py 프로젝트: owaisj/portal
 def ready(self):
     from elasticsearch_dsl.connections import connections
     from django.conf import settings
     try:
         connections.configure(default={
             'hosts':
             settings.ES_CONNECTIONS[settings.DESIGNSAFE_ENVIRONMENT]
             ['hosts'],
             "http_auth":
             settings.ES_AUTH
         },
                               request_timeout=60,
                               sniff_on_start=True,
                               sniffer_timeout=60,
                               sniff_on_connection_fail=True,
                               sniff_timeout=10,
                               max_retries=3,
                               retry_on_timeout=True)
     except AttributeError as exc:
         logger.error('Missing ElasticSearch config. %s', exc)
         raise
     from designsafe.apps.data.models.agave.base import set_lazy_rels
     from designsafe.apps.projects.models.agave.experimental import *
     from designsafe.apps.projects.models.agave.simulation import *
     from designsafe.apps.projects.models.agave.hybrid_simulation import *
     from designsafe.apps.projects.models.agave.rapid import *
     set_lazy_rels()
     super(DataConfig, self).ready()
예제 #6
0
파일: apps.py 프로젝트: bluemini/kuma
    def ready(self):
        super(WikiConfig, self).ready()

        # Configure Elasticsearch connections for connection pooling.
        es_connections.configure(
            default={
                'hosts': settings.ES_URLS,
            },
            indexing={
                'hosts': settings.ES_URLS,
                'timeout': settings.ES_INDEXING_TIMEOUT,
            },
        )

        # connect some signal handlers for the wiki models
        Document = self.get_model('Document')
        signals.post_save.connect(self.on_document_save,
                                  sender=Document,
                                  dispatch_uid='wiki.document.post_save')
        render_done.connect(self.on_render_done,
                            dispatch_uid='wiki.document.render_done')

        Revision = self.get_model('Revision')
        signals.post_save.connect(self.on_revision_save,
                                  sender=Revision,
                                  dispatch_uid='wiki.revision.post_save')

        DocumentZone = self.get_model('DocumentZone')
        signals.post_save.connect(self.on_zone_save,
                                  sender=DocumentZone,
                                  dispatch_uid='wiki.zone.post_save')
예제 #7
0
 def ready(self):
     connections.configure(**settings.ES_CONNECTIONS)
     try:
         # FIXME: this shouldn't be run before index_data!
         Question._doc_type.refresh()
         Answer._doc_type.refresh()
     except NotFoundError:
         pass
예제 #8
0
 def ready(self):
     connections.configure(**settings.ES_CONNECTIONS)
     try:
         # FIXME: this shouldn't be run before index_data!
         Question._doc_type.refresh()
         Answer._doc_type.refresh()
     except NotFoundError:
         pass
예제 #9
0
def get_config():
    global CONFIG
    if CONFIG is None:
        with open(CONFIG_FILE) as f:
            CONFIG = json.load(f)
    es_config = CONFIG["ELASTICSEARCH"]
    connections.configure(default={"hosts": es_config["HOSTS"], "timeout": es_config["TIMEOUT"]})
    return CONFIG
예제 #10
0
    def ready(self):
        if 'HOST' in self.settings:
            raise NotImplementedError('"HOST" key replaced by "CONNECTIONS"')
        options = {}
        for alias, details in self.settings['CONNECTIONS'].items():
            options[alias] = details

        connections.configure(**options)
예제 #11
0
    def ready(self):
        if 'HOST' in self.settings:
            raise NotImplementedError('"HOST" key replaced by "CONNECTIONS"')
        options = {}
        for alias, details in self.settings['CONNECTIONS'].items():
            options[alias] = details

        connections.configure(**options)
예제 #12
0
 def setUpClass(cls):
     user = settings.GITHUB['user']
     passwd = settings.GITHUB['passwd']
     cls.github = Github(user, passwd)
     cls.faker = Faker('it_IT')
     repo_name = settings.GITHUB['repo']
     cls.repo = cls.github.get_repo(repo_name)
     connections.configure(**settings.ELASTICSEARCH)
     cls.es = connections.get_connection()
예제 #13
0
파일: apps.py 프로젝트: marcinn/springy
    def ready(self):
        from elasticsearch_dsl.connections import connections
        from .settings import DATABASES, AUTODISCOVER_MODULE, AUTODISCOVER
        from .utils import autodiscover

        connections.configure(**DATABASES)

        if AUTODISCOVER:
            autodiscover(AUTODISCOVER_MODULE)
예제 #14
0
 def set_up(self):
     connections.configure(
         default={
             'hosts': settings.ELASTICSEARCH,
             'verify_certs': True,
             'ca_certs': certifi.where(),
             'timeout': 60.0,
         },
     )
예제 #15
0
 def ready(self):
     super(DocsConfig, self).ready()
     # Configure Elasticsearch connections for connection pooling.
     connections.configure(default={
         'hosts': settings.ES_HOST,
         'verify_certs': True,
         'ca_certs': certifi.where(),
         'timeout': 60.0,
     }, )
예제 #16
0
 def ready(self):
     super(DocsConfig, self).ready()
     # Configure Elasticsearch connections for connection pooling.
     connections.configure(
         default={
             'hosts': settings.ES_HOST,
             'verify_certs': True,
             'ca_certs': certifi.where(),
         },
     )
예제 #17
0
파일: apps.py 프로젝트: mift-2020/mift
 def ready(self):
     self.module.autodiscover()
     connections.configure(**settings.ELASTICSEARCH_DSL)
     # Setup the signal processor.
     if not self.signal_processor:
         signal_processor_path = getattr(
             settings, 'ELASTICSEARCH_DSL_SIGNAL_PROCESSOR',
             'django_elasticsearch_dsl.signals.RealTimeSignalProcessor')
         signal_processor_class = import_string(signal_processor_path)
         self.signal_processor = signal_processor_class(connections)
예제 #18
0
def es_conn(server=settings.ES_SERVER):
    """Standardized connection to the ES cluster.

    :param server: a server definition of the form [host:port, ...].  See
    https://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch
    for alternate host specification options.
    :return: an Elasticsearch connection instance
    """

    connections.configure(default=server, max_retries=1, sniff_on_start=False)
    return connections.get_connection()
예제 #19
0
def es_conn(server=settings.ES_SERVER):
    """Standardized connection to the ES cluster.

    :param server: a server definition of the form [host:port, ...].  See
    https://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch
    for alternate host specification options.
    :return: an Elasticsearch connection instance
    """

    connections.configure(default=server, max_retries=1, sniff_on_start=False)
    return connections.get_connection()
예제 #20
0
def _write_es(path):
    settings = {
        "default": {
            "hosts": [{
                "host": "127.0.0.1",
                "port": 9200
            }],
            "max_retries": 0,
            "timeout": 120
        }
    }
    connections.configure(**settings)

    request_body = {
        "settings": {},
        'mappings': {
            'properties': {
                'county': {
                    'type': 'keyword'
                },
                'city': {
                    'type': 'keyword'
                },
                'vendor': {
                    'type': 'keyword'
                },
                'product': {
                    'type': 'keyword'
                },
                'device_type': {
                    'type': 'keyword'
                },
            }
        }
    }
    es = connections.get_connection()
    index_name = 'some-index'
    try:
        es.indices.get(index=index_name)
        es.indices.delete(index=index_name)
    except NotFoundError:
        pass
    es.indices.create(index=index_name, body=request_body)

    for each in glob.glob(os.path.join(path, '*.json')):
        try:
            helpers.bulk(connections.get_connection(),
                         load_json(each),
                         index=index_name)
        except Exception as e:
            print(f'{each} insert failed {e}')
            continue
예제 #21
0
    def ready(self):
        """Configure kuma.search after models are loaded."""
        # Register signal handlers
        from . import signal_handlers  # noqa

        # Configure Elasticsearch connections for connection pooling.
        es_connections.configure(
            default={"hosts": settings.ES_URLS},
            indexing={
                "hosts": settings.ES_URLS,
                "timeout": settings.ES_INDEXING_TIMEOUT,
            },
        )
예제 #22
0
 def register(self, model, index):
     """Register the model with the registry"""
     self.model_to_indexes[model].add(index)
     if not self.connected:
         connections.index_name = {}
         from django.conf import settings
         kwargs = {}
         for name, params in settings.ELASTICSEARCH_CONNECTIONS.items():
             params = copy.deepcopy(params)
             kwargs[name] = params
             connections.index_name[name] = params.pop("index_name")
         connections.configure(**kwargs)
         self.connected = True
예제 #23
0
 def register(self, model, index):
     """Register the model with the registry"""
     self.model_to_indexes[model].add(index)
     if not self.connected:
         connections.index_name = {}
         from django.conf import settings
         kwargs = {}
         for name, params in settings.ELASTICSEARCH_CONNECTIONS.items():
             params = copy.deepcopy(params)
             kwargs[name] = params
             connections.index_name[name] = params.pop("index_name")
         connections.configure(**kwargs)
         self.connected = True
예제 #24
0
 def ready(self):
     super(DocsConfig, self).ready()
     # Configure Elasticsearch connections for connection pooling.
     connections.configure(
         default={
             'hosts': settings.ES_HOST,
             'verify_certs': True,
             # We need to use old_where as long as our openssl version is below 1.0.2
             # and ES found uses a CA which is cross signed by a 1024-bit root.
             'ca_certs': certifi.old_where(),
             'timeout': 60.0,
         },
     )
예제 #25
0
 def ready(self):
     super().ready()
     # Configure Elasticsearch connections for connection pooling.
     connections.configure(
         default={
             'hosts': settings.ES_HOST,
             'verify_certs': True,
             # We need to use old_where as long as our openssl version is below 1.0.2
             # and ES found uses a CA which is cross signed by a 1024-bit root.
             'ca_certs': certifi.old_where(),
             'timeout': 60.0,
         },
     )
예제 #26
0
def get_config():
    global CONFIG
    if CONFIG is None:
        with open(CONFIG_FILE) as f:
            CONFIG = json.load(f)
        es_config = CONFIG["ELASTICSEARCH"]
        connections.configure(
            default={
                "hosts": es_config["HOSTS"],
                "timeout": es_config["TIMEOUT"],
                "max_retries": es_config["MAX_RETRIES"],
                "retry_on_timeout": es_config["RETRY_ON_TIMEOUT"],
            })
    return CONFIG
예제 #27
0
파일: apps.py 프로젝트: MatonAnthony/kuma
    def ready(self):
        """Configure kuma.search after models are loaded."""
        # Register signal handlers
        from . import signal_handlers  # noqa

        # Configure Elasticsearch connections for connection pooling.
        es_connections.configure(
            default={
                'hosts': settings.ES_URLS,
            },
            indexing={
                'hosts': settings.ES_URLS,
                'timeout': settings.ES_INDEXING_TIMEOUT,
            },
        )
예제 #28
0
def get_es():
    """Return an es connection or None if none seems available.

    Also wait for ES to be ready (yellow status)
    """
    # you may use ES_HOST environment variable to configure Elasticsearch
    # launching something like
    # docker run --rm -p "127.0.0.1:9200:9200" -e "discovery.type=single-node" elasticsearch:7.8.0
    # is a simple way to get an instance
    connections.configure(
        default=dict(hosts=os.environ.get("ES_HOST", "localhost"), timeout=20))
    client = connections.get_connection("default")
    try:
        # check ES running
        client.cluster.health(wait_for_status='yellow')
    except ConnectionError:
        client = None
    return client
예제 #29
0
def configure_connections():
    """
    Create connections for the application

    This should only be called once
    """
    # this is the default connection
    http_auth = settings.ELASTICSEARCH_HTTP_AUTH
    use_ssl = http_auth is not None
    # configure() lazily creates connections when get_connection() is called
    connections.configure(
        default={
            "hosts": [settings.ELASTICSEARCH_URL],
            "http_auth": http_auth,
            "use_ssl": use_ssl,
            # make sure we verify SSL certificates (off by default)
            "verify_certs": use_ssl,
        })
예제 #30
0
    def handle(self, **options):
        """
        Entry point for the command.

        Args:
            options: The CLI arguments of the command.
        """
        action = options['action']
        models = self._get_models(options['models'])
        connections.configure(**ELASTICSEARCH_DSL)
        connection = connections.get_connection(options['using'])

        try:
            func = getattr(self, 'handle_{}'.format(action))
        except AttributeError:
            raise CommandError("Invalid action {}. Must be one of: {}".format(
                action, ', '.join(self.possible_actions),
            ))

        func(models, options, connection)
예제 #31
0
 def ready(self):
     self.module.autodiscover()
     if getattr(settings, 'ELASTICSEARCH_DSL', None):
         connections.configure(**settings.ELASTICSEARCH_DSL)
         es_scheduler = {
             'create snapshot every day at midnight': {
                 'task': 'Snapshot',
                 'schedule': crontab(hour=0, minute=0),
                 'args': ('daily', )
             },
             'create snapshot every first day of month at midnight': {
                 'task': 'Snapshot',
                 'schedule': crontab(hour=0, minute=0, day_of_month=1),
                 'args': ('monthly', )
             },
         }
         if not app.conf.beat_schedule:
             app.conf.beat_schedule = es_scheduler
         else:
             app.conf.beat_schedule.update(es_scheduler)
예제 #32
0
    def handle(self, **options):
        """
        Entry point for the command.

        Args:
            options: The CLI arguments of the command.
        """
        action = options['action']
        models = self._get_models(options['models'])
        connections.configure(**ELASTICSEARCH_DSL)
        connection = connections.get_connection(options['using'])

        try:
            func = getattr(self, 'handle_{}'.format(action))
        except AttributeError:
            raise CommandError("Invalid action {}. Must be one of: {}".format(
                action,
                ', '.join(self.possible_actions),
            ))

        func(models, options, connection)
예제 #33
0
파일: apps.py 프로젝트: riverspirit/kuma
    def ready(self):
        super(WikiConfig, self).ready()

        # Configure Elasticsearch connections for connection pooling.
        es_connections.configure(
            default={"hosts": settings.ES_URLS},
            indexing={"hosts": settings.ES_URLS, "timeout": settings.ES_INDEXING_TIMEOUT},
        )

        # connect some signal handlers for the wiki models
        Document = self.get_model("Document")
        signals.post_save.connect(self.on_document_save, sender=Document, dispatch_uid="wiki.document.post_save")
        render_done.connect(self.on_render_done, dispatch_uid="wiki.document.render_done")

        Revision = self.get_model("Revision")
        signals.post_save.connect(self.on_revision_save, sender=Revision, dispatch_uid="wiki.revision.post_save")

        DocumentZone = self.get_model("DocumentZone")
        signals.post_save.connect(self.on_zone_save, sender=DocumentZone, dispatch_uid="wiki.zone.post_save")

        DocumentSpamAttempt = self.get_model("DocumentSpamAttempt")
        signals.post_save.connect(
            self.on_document_spam_attempt_save, sender=DocumentSpamAttempt, dispatch_uid="wiki.spam_attempt.post_save"
        )
예제 #34
0
class DJESConfig(AppConfig):
    name = 'djes'
    verbose_name = "DJ E.S."

    def ready(self):
        from .models import Indexable

        # Let's register all the Indexable models
        for model in apps.get_models():

            # If it quacks...
            if issubclass(model, Indexable):
                meta = getattr(model, "_meta")
                if meta and not getattr(meta, "abstract"):
                    indexable_registry.register(model)

    connections.configure(**settings.ES_CONNECTIONS)
예제 #35
0
#!/usr/bin/env python3
import sys
import fileinput
import json
from elasticsearch_dsl import Document, field, InnerDoc
from elasticsearch_dsl.connections import connections
import logging

logger = logging.getLogger('chibi_dl_elasticsearch.models.nhentai')

connections.configure(default={'host': 'waifus', 'port': 80})

from elasticsearch_dsl import analyzer, tokenizer

tag = analyzer(
    'tag',
    tokenizer=tokenizer('trigram', 'nGram', min_gram=3, max_gram=3),
    filter=[
        "lowercase",
    ],
)

titles = analyzer(
    'titles',
    tokenizer=tokenizer('trigram', 'nGram', min_gram=3, max_gram=5),
    filter=[
        "lowercase",
    ],
)

titles_space = analyzer(
########################################################



# Default connection

from elasticsearch_dsl.connections import connections

connections.create_connection(hosts=['localhost'], timeout=20)

###############################################
# Multiple clusters
connections.configure(
        default={'hosts': 'localhost'},
        dev={
            'hosts': ['esdev1.example.com:9200'],
            'sniff_on_start': True
            }
        )

# We can also add them later using aliases...
connections.create_connection('qa', hosts=['esqa1.example.com'], sniff_on_start=True)

connections.add_connection('qa', my_client)

# When searching we can refer to a specific connection by using aliases
s = Search(using='qa')

###############################################
# The api is chainable
s = Search().using(client).query('match', title='python')
예제 #37
0
# coding: utf-8
from elasticsearch_dsl.connections import connections
ES_HOST = "10.10.0.247:9200"
# ES_HOST = {"host": "192.168.10.190:9200"}
connections.configure(default={"hosts": ES_HOST})
REDIS_HOST = ""
REDIS_PASSWORD = ""

# REDIS_HOST = "127.0.0.1"
# REDIS_PASSWORD = ""

ES_INDEX = "mxx_zjac_t2"
ES_DOC_TYPE = "doc"

def configure():
    connections_definitions = getattr(settings, 'FLEXY_CONNECTIONS', {})
    connections.configure(**connections_definitions)
예제 #39
0
파일: apps.py 프로젝트: HelloLily/hellolily
 def ready(self):
     autodiscover_modules('documents')
     connections.configure(**settings.ELASTICSEARCH_DSL)
예제 #40
0
파일: apps.py 프로젝트: dboczek/springy
 def ready(self):
     from .settings import DATABASES
     from elasticsearch_dsl.connections import connections
     connections.configure(**DATABASES)
예제 #41
0
from elasticsearch_dsl.query import Q
from elasticsearch_dsl.connections import connections
from .base import BaseFileManager

logger = logging.getLogger(__name__)

try:
    es_settings = getattr(settings, 'ELASTIC_SEARCH', {})
    published_index = es_settings['published_index']
    cluster = es_settings['cluster']
    hosts = cluster['hosts']
    connections.configure(
        default={
            'hosts': hosts,
            'sniff_on_start': True,
            'sniff_on_connection_fail': True,
            'sniffer_timeout': 60,
            'retry_on_timeout': True,
            'timeout:': 20,
        })
except KeyError as e:
    logger.exception('ELASTIC_SEARCH missing %s' % e)

class PublicProjectIndexed(DocType):
    class Meta:
        index = published_index
        doc_type = 'project'

class PublicExperimentIndexed(DocType):
    class Meta:
        index = published_index
예제 #42
0
from elasticsearch_dsl.connections import connections
from elasticsearch_dsl import DocType, Text, Date, Search
from elasticsearch.helpers import bulk
from elasticsearch import Elasticsearch
from . import models

connections.configure(default={'hosts': ['elk:9200'], 'sniff_on_start': True})
connections.create_connection(hosts='elk:9200')


def bulk_indexing():
    BlogPostIndex.init()
    es = Elasticsearch(['elk:9200'])
    bulk(client=es,
         actions=(b.indexing()
                  for b in models.BlogPost.objects.all().iterator()))


class BlogPostIndex(DocType):
    author = Text()
    posted_date = Date()
    title = Text()
    text = Text()

    class Meta:
        index = 'blogpost-index'


def search(author):
    s = Search().filter('term', author=author)
    response = s.execute()
#使用该方法不能指定_id
from elasticsearch import Elasticsearch
from sqlalchemy import create_engine
from elasticsearch_dsl import Document, Text, Integer, Keyword, Long, Boolean
from elasticsearch_dsl.connections import connections

connections.configure(default={'hosts': '192.168.59.129:9200'})


class TeacherType(Document):
    tno = Long(required=True)
    tname = Keyword()
    tsex = Keyword()
    tbirthday = Keyword()
    prof = Keyword()
    depart = Text(analyzer="ik_max_word")

    class Index():
        name = 'teacher'
        # doc_type = 'teacher'


class Mysql_Data_to_Es():
    def __init__(self):
        self.db = create_engine(
            "mysql+pymysql://root:[email protected]:3306/sql_test",
            encoding='utf8')

    def get_mysql_date(self):
        resultProxy = self.db.execute('select * from teacher')
        data = resultProxy.fetchall()
예제 #44
0
    except:
        print("ERROR: Did you remember to generate config files with credstmpl?", file=sys.stderr)
        print("Check out credstmpl at https://github.com/qadium/credstmpl", file=sys.stderr)
        print("you'll need to run `credstmpl filename.extension.j2`", file=sys.stderr)

def es_config(**kwargs):
    return dict(
        verify_certs=False,
        use_ssl=False,
        request_timeout=160,
        timeout=160,
        **kwargs
    )

def get_config():
    env = os.getenv('PRODUCTION', None)
    cfg = load_config('production') if env else load_config('develop')

    cfg.update(load_config("common"))

    logger.debug("Using {env} environment.".format(env=env))

    return cfg

cfg = get_config()

connections.configure(
        cdr=es_config(**cfg['cdr_elastic_search']),
        local=es_config(hosts='localhost', index='factor_state2016'),
)
예제 #45
0
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from elasticsearch_dsl.connections import connections

default_app_config = 'search.apps.SearchAppConfig'

SEARCH_CONFIG = getattr(settings, 'SEARCH', None)
if not SEARCH_CONFIG:
    raise ImproperlyConfigured('SEARCH settings dictionary must be provided')
if 'default' not in SEARCH_CONFIG:
    raise ImproperlyConfigured("SEARCH settings must include 'default' settings dict, similar to DATABASES")
if 'index' not in SEARCH_CONFIG['default']:
    raise ImproperlyConfigured("SEARCH settings must include a name for 'index' in 'default' settings dict")

# Set up ES connection
connection_settings = SEARCH_CONFIG['default'].get('connections')
if not connection_settings:
    raise ImproperlyConfigured("SEARCH settings must include 'connections' dict in 'default'")
connections.configure(default=connection_settings)
예제 #46
0
파일: search.py 프로젝트: Osmose/kuma
from elasticsearch.helpers import bulk
from elasticsearch_dsl import document, field
from elasticsearch_dsl.connections import connections
from elasticsearch_dsl.mapping import Mapping
from elasticsearch_dsl.search import Search

from kuma.core.utils import chord_flow, chunked


log = logging.getLogger('kuma.wiki.search')


# Configure Elasticsearch connections for connection pooling.
connections.configure(
    default={'hosts': settings.ES_URLS},
    indexing={'hosts': settings.ES_URLS,
              'timeout': settings.ES_INDEXING_TIMEOUT},
)


class WikiDocumentType(document.DocType):
    excerpt_fields = ['summary', 'content']
    exclude_slugs = ['Talk:', 'User:'******'User_talk:', 'Template_talk:',
                     'Project_talk:']

    boost = field.Float(null_value=1.0)
    content = field.String(analyzer='kuma_content',
                           term_vector='with_positions_offsets')
    css_classnames = field.String(analyzer='case_insensitive_keyword')
    html_attributes = field.String(analyzer='case_insensitive_keyword')
    id = field.Long()
예제 #47
0
파일: apps.py 프로젝트: Givemore/fjord
    def ready(self):
        # Set up Elasticsearch connections based on settings
        urls = getattr(settings, 'ES_URLS', DEFAULT_URLS)
        timeout = getattr(settings, 'ES_TIMEOUT', DEFAULT_TIMEOUT)

        connections.configure(default={'hosts': urls, 'timeout': timeout})
예제 #48
0
파일: import.py 프로젝트: lang-uk/names-ms
        pass


def normalize_alphabet(rec):
    rec["term"] = normalize_charset(rec["term"])
    return rec

if __name__ == '__main__':
    if len(sys.argv) < 2:
        raise Exception("Input file argument is not specified")

    input_fname = sys.argv[1]
    if not os.path.exists(input_fname):
        raise Exception("Input file doesn't exist")

    connections.configure(**ELASTICSEARCH_CONNECTIONS)
    Index(NameVariant._doc_type.index).delete(ignore=404)
    NameVariant.init()

    accum = []
    total_lines = 0
    with open(input_fname, encoding="utf-8") as input_fp:
        for i, line in enumerate(input_fp):
            accum.append(loads(line))
            if len(accum) >= 10000:
                total_lines += len(accum)
                bulk_load(map(normalize_alphabet, accum))
                accum = []
                print("Loaded: %s items" % total_lines)

        bulk_load(accum)
예제 #49
0
def createConnection(host, timeout):
    connections.configure(
        default={'hosts': host, 'timeout': timeout},
    )
    es = Elasticsearch()
    return es
예제 #50
0
MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'seeker',
    'core',
)

ROOT_URLCONF = 'urls'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': ':memory:',
    }
}

LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

SEEKER_INDEX = 'seeker-tests'

from elasticsearch_dsl.connections import connections
connections.configure(default={'hosts': 'localhost'})
예제 #51
0
 def ready(self):
     connections.configure(**settings.ELASTICSEARCH_DSL)
     autodiscover_modules("metrics")
 def ready(self):
     connections.configure(**settings.ES_CONNECTIONS)