def configure_connection(): """Configure Elasticsearch default connection.""" connections_default = { 'hosts': [settings.ES_URL], 'verify_certs': settings.ES_VERIFY_CERTS, } connections.configure(default=connections_default)
def ready(self): super(WikiConfig, self).ready() # Configure Elasticsearch connections for connection pooling. es_connections.configure( default={ 'hosts': settings.ES_URLS, }, indexing={ 'hosts': settings.ES_URLS, 'timeout': settings.ES_INDEXING_TIMEOUT, }, ) # connect some signal handlers for the wiki models Document = self.get_model('Document') signals.post_save.connect(self.on_document_save, sender=Document, dispatch_uid='wiki.document.post_save') render_done.connect(self.on_render_done, dispatch_uid='wiki.document.render_done') Revision = self.get_model('Revision') signals.post_save.connect(self.on_revision_save, sender=Revision, dispatch_uid='wiki.revision.post_save') DocumentZone = self.get_model('DocumentZone') signals.post_save.connect(self.on_zone_save, sender=DocumentZone, dispatch_uid='wiki.zone.post_save')
def setUpClass(cls): if cls._overridden_settings: cls._cls_overridden_context = override_settings( **cls._overridden_settings) cls._cls_overridden_context.enable() connections.configure(**settings.ELASTICSEARCH_CONNECTIONS) cls.es_client = cls._get_client() IngestClient(cls.es_client).put_pipeline( id='ingest_attachment', body={ 'description': "Extract attachment information", 'processors': [{ "attachment": { "field": "data", "indexed_chars": "-1" }, "remove": { "field": "data" } }] }) super().setUpClass()
def configure_connection(): """Configure Elasticsearch default connection.""" if settings.ES_USE_AWS_AUTH: es_protocol = { 'http': 80, 'https': 443, } es_host = urlparse(settings.ES_URL) es_port = es_host.port if es_host.port else es_protocol.get( es_host.scheme) auth = AWSRequestsAuth( aws_access_key=settings.AWS_ELASTICSEARCH_KEY, aws_secret_access_key=settings.AWS_ELASTICSEARCH_SECRET, aws_host=es_host.netloc, aws_region=settings.AWS_ELASTICSEARCH_REGION, aws_service='es', ) connections_default = { 'hosts': [es_host.netloc], 'port': es_port, 'use_ssl': settings.ES_USE_SSL, 'verify_certs': settings.ES_VERIFY_CERTS, 'http_auth': auth, 'connection_class': RequestsHttpConnection, } else: connections_default = { 'hosts': [settings.ES_URL], 'verify_certs': settings.ES_VERIFY_CERTS, } connections.configure(default=connections_default, )
def ready(self): from elasticsearch_dsl.connections import connections from django.conf import settings try: connections.configure(default={ 'hosts': settings.ES_CONNECTIONS[settings.DESIGNSAFE_ENVIRONMENT] ['hosts'], "http_auth": settings.ES_AUTH }, request_timeout=60, sniff_on_start=True, sniffer_timeout=60, sniff_on_connection_fail=True, sniff_timeout=10, max_retries=3, retry_on_timeout=True) except AttributeError as exc: logger.error('Missing ElasticSearch config. %s', exc) raise from designsafe.apps.data.models.agave.base import set_lazy_rels from designsafe.apps.projects.models.agave.experimental import * from designsafe.apps.projects.models.agave.simulation import * from designsafe.apps.projects.models.agave.hybrid_simulation import * from designsafe.apps.projects.models.agave.rapid import * set_lazy_rels() super(DataConfig, self).ready()
def ready(self): super(WikiConfig, self).ready() # Configure Elasticsearch connections for connection pooling. es_connections.configure( default={ 'hosts': settings.ES_URLS, }, indexing={ 'hosts': settings.ES_URLS, 'timeout': settings.ES_INDEXING_TIMEOUT, }, ) # connect some signal handlers for the wiki models Document = self.get_model('Document') signals.post_save.connect(self.on_document_save, sender=Document, dispatch_uid='wiki.document.post_save') render_done.connect(self.on_render_done, dispatch_uid='wiki.document.render_done') Revision = self.get_model('Revision') signals.post_save.connect(self.on_revision_save, sender=Revision, dispatch_uid='wiki.revision.post_save') DocumentZone = self.get_model('DocumentZone') signals.post_save.connect(self.on_zone_save, sender=DocumentZone, dispatch_uid='wiki.zone.post_save')
def ready(self): connections.configure(**settings.ES_CONNECTIONS) try: # FIXME: this shouldn't be run before index_data! Question._doc_type.refresh() Answer._doc_type.refresh() except NotFoundError: pass
def ready(self): connections.configure(**settings.ES_CONNECTIONS) try: # FIXME: this shouldn't be run before index_data! Question._doc_type.refresh() Answer._doc_type.refresh() except NotFoundError: pass
def get_config(): global CONFIG if CONFIG is None: with open(CONFIG_FILE) as f: CONFIG = json.load(f) es_config = CONFIG["ELASTICSEARCH"] connections.configure(default={"hosts": es_config["HOSTS"], "timeout": es_config["TIMEOUT"]}) return CONFIG
def ready(self): if 'HOST' in self.settings: raise NotImplementedError('"HOST" key replaced by "CONNECTIONS"') options = {} for alias, details in self.settings['CONNECTIONS'].items(): options[alias] = details connections.configure(**options)
def ready(self): if 'HOST' in self.settings: raise NotImplementedError('"HOST" key replaced by "CONNECTIONS"') options = {} for alias, details in self.settings['CONNECTIONS'].items(): options[alias] = details connections.configure(**options)
def setUpClass(cls): user = settings.GITHUB['user'] passwd = settings.GITHUB['passwd'] cls.github = Github(user, passwd) cls.faker = Faker('it_IT') repo_name = settings.GITHUB['repo'] cls.repo = cls.github.get_repo(repo_name) connections.configure(**settings.ELASTICSEARCH) cls.es = connections.get_connection()
def ready(self): from elasticsearch_dsl.connections import connections from .settings import DATABASES, AUTODISCOVER_MODULE, AUTODISCOVER from .utils import autodiscover connections.configure(**DATABASES) if AUTODISCOVER: autodiscover(AUTODISCOVER_MODULE)
def set_up(self): connections.configure( default={ 'hosts': settings.ELASTICSEARCH, 'verify_certs': True, 'ca_certs': certifi.where(), 'timeout': 60.0, }, )
def ready(self): super(DocsConfig, self).ready() # Configure Elasticsearch connections for connection pooling. connections.configure(default={ 'hosts': settings.ES_HOST, 'verify_certs': True, 'ca_certs': certifi.where(), 'timeout': 60.0, }, )
def ready(self): super(DocsConfig, self).ready() # Configure Elasticsearch connections for connection pooling. connections.configure( default={ 'hosts': settings.ES_HOST, 'verify_certs': True, 'ca_certs': certifi.where(), }, )
def ready(self): self.module.autodiscover() connections.configure(**settings.ELASTICSEARCH_DSL) # Setup the signal processor. if not self.signal_processor: signal_processor_path = getattr( settings, 'ELASTICSEARCH_DSL_SIGNAL_PROCESSOR', 'django_elasticsearch_dsl.signals.RealTimeSignalProcessor') signal_processor_class = import_string(signal_processor_path) self.signal_processor = signal_processor_class(connections)
def es_conn(server=settings.ES_SERVER): """Standardized connection to the ES cluster. :param server: a server definition of the form [host:port, ...]. See https://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch for alternate host specification options. :return: an Elasticsearch connection instance """ connections.configure(default=server, max_retries=1, sniff_on_start=False) return connections.get_connection()
def es_conn(server=settings.ES_SERVER): """Standardized connection to the ES cluster. :param server: a server definition of the form [host:port, ...]. See https://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch for alternate host specification options. :return: an Elasticsearch connection instance """ connections.configure(default=server, max_retries=1, sniff_on_start=False) return connections.get_connection()
def _write_es(path): settings = { "default": { "hosts": [{ "host": "127.0.0.1", "port": 9200 }], "max_retries": 0, "timeout": 120 } } connections.configure(**settings) request_body = { "settings": {}, 'mappings': { 'properties': { 'county': { 'type': 'keyword' }, 'city': { 'type': 'keyword' }, 'vendor': { 'type': 'keyword' }, 'product': { 'type': 'keyword' }, 'device_type': { 'type': 'keyword' }, } } } es = connections.get_connection() index_name = 'some-index' try: es.indices.get(index=index_name) es.indices.delete(index=index_name) except NotFoundError: pass es.indices.create(index=index_name, body=request_body) for each in glob.glob(os.path.join(path, '*.json')): try: helpers.bulk(connections.get_connection(), load_json(each), index=index_name) except Exception as e: print(f'{each} insert failed {e}') continue
def ready(self): """Configure kuma.search after models are loaded.""" # Register signal handlers from . import signal_handlers # noqa # Configure Elasticsearch connections for connection pooling. es_connections.configure( default={"hosts": settings.ES_URLS}, indexing={ "hosts": settings.ES_URLS, "timeout": settings.ES_INDEXING_TIMEOUT, }, )
def register(self, model, index): """Register the model with the registry""" self.model_to_indexes[model].add(index) if not self.connected: connections.index_name = {} from django.conf import settings kwargs = {} for name, params in settings.ELASTICSEARCH_CONNECTIONS.items(): params = copy.deepcopy(params) kwargs[name] = params connections.index_name[name] = params.pop("index_name") connections.configure(**kwargs) self.connected = True
def register(self, model, index): """Register the model with the registry""" self.model_to_indexes[model].add(index) if not self.connected: connections.index_name = {} from django.conf import settings kwargs = {} for name, params in settings.ELASTICSEARCH_CONNECTIONS.items(): params = copy.deepcopy(params) kwargs[name] = params connections.index_name[name] = params.pop("index_name") connections.configure(**kwargs) self.connected = True
def ready(self): super(DocsConfig, self).ready() # Configure Elasticsearch connections for connection pooling. connections.configure( default={ 'hosts': settings.ES_HOST, 'verify_certs': True, # We need to use old_where as long as our openssl version is below 1.0.2 # and ES found uses a CA which is cross signed by a 1024-bit root. 'ca_certs': certifi.old_where(), 'timeout': 60.0, }, )
def ready(self): super().ready() # Configure Elasticsearch connections for connection pooling. connections.configure( default={ 'hosts': settings.ES_HOST, 'verify_certs': True, # We need to use old_where as long as our openssl version is below 1.0.2 # and ES found uses a CA which is cross signed by a 1024-bit root. 'ca_certs': certifi.old_where(), 'timeout': 60.0, }, )
def get_config(): global CONFIG if CONFIG is None: with open(CONFIG_FILE) as f: CONFIG = json.load(f) es_config = CONFIG["ELASTICSEARCH"] connections.configure( default={ "hosts": es_config["HOSTS"], "timeout": es_config["TIMEOUT"], "max_retries": es_config["MAX_RETRIES"], "retry_on_timeout": es_config["RETRY_ON_TIMEOUT"], }) return CONFIG
def ready(self): """Configure kuma.search after models are loaded.""" # Register signal handlers from . import signal_handlers # noqa # Configure Elasticsearch connections for connection pooling. es_connections.configure( default={ 'hosts': settings.ES_URLS, }, indexing={ 'hosts': settings.ES_URLS, 'timeout': settings.ES_INDEXING_TIMEOUT, }, )
def get_es(): """Return an es connection or None if none seems available. Also wait for ES to be ready (yellow status) """ # you may use ES_HOST environment variable to configure Elasticsearch # launching something like # docker run --rm -p "127.0.0.1:9200:9200" -e "discovery.type=single-node" elasticsearch:7.8.0 # is a simple way to get an instance connections.configure( default=dict(hosts=os.environ.get("ES_HOST", "localhost"), timeout=20)) client = connections.get_connection("default") try: # check ES running client.cluster.health(wait_for_status='yellow') except ConnectionError: client = None return client
def configure_connections(): """ Create connections for the application This should only be called once """ # this is the default connection http_auth = settings.ELASTICSEARCH_HTTP_AUTH use_ssl = http_auth is not None # configure() lazily creates connections when get_connection() is called connections.configure( default={ "hosts": [settings.ELASTICSEARCH_URL], "http_auth": http_auth, "use_ssl": use_ssl, # make sure we verify SSL certificates (off by default) "verify_certs": use_ssl, })
def handle(self, **options): """ Entry point for the command. Args: options: The CLI arguments of the command. """ action = options['action'] models = self._get_models(options['models']) connections.configure(**ELASTICSEARCH_DSL) connection = connections.get_connection(options['using']) try: func = getattr(self, 'handle_{}'.format(action)) except AttributeError: raise CommandError("Invalid action {}. Must be one of: {}".format( action, ', '.join(self.possible_actions), )) func(models, options, connection)
def ready(self): self.module.autodiscover() if getattr(settings, 'ELASTICSEARCH_DSL', None): connections.configure(**settings.ELASTICSEARCH_DSL) es_scheduler = { 'create snapshot every day at midnight': { 'task': 'Snapshot', 'schedule': crontab(hour=0, minute=0), 'args': ('daily', ) }, 'create snapshot every first day of month at midnight': { 'task': 'Snapshot', 'schedule': crontab(hour=0, minute=0, day_of_month=1), 'args': ('monthly', ) }, } if not app.conf.beat_schedule: app.conf.beat_schedule = es_scheduler else: app.conf.beat_schedule.update(es_scheduler)
def handle(self, **options): """ Entry point for the command. Args: options: The CLI arguments of the command. """ action = options['action'] models = self._get_models(options['models']) connections.configure(**ELASTICSEARCH_DSL) connection = connections.get_connection(options['using']) try: func = getattr(self, 'handle_{}'.format(action)) except AttributeError: raise CommandError("Invalid action {}. Must be one of: {}".format( action, ', '.join(self.possible_actions), )) func(models, options, connection)
def ready(self): super(WikiConfig, self).ready() # Configure Elasticsearch connections for connection pooling. es_connections.configure( default={"hosts": settings.ES_URLS}, indexing={"hosts": settings.ES_URLS, "timeout": settings.ES_INDEXING_TIMEOUT}, ) # connect some signal handlers for the wiki models Document = self.get_model("Document") signals.post_save.connect(self.on_document_save, sender=Document, dispatch_uid="wiki.document.post_save") render_done.connect(self.on_render_done, dispatch_uid="wiki.document.render_done") Revision = self.get_model("Revision") signals.post_save.connect(self.on_revision_save, sender=Revision, dispatch_uid="wiki.revision.post_save") DocumentZone = self.get_model("DocumentZone") signals.post_save.connect(self.on_zone_save, sender=DocumentZone, dispatch_uid="wiki.zone.post_save") DocumentSpamAttempt = self.get_model("DocumentSpamAttempt") signals.post_save.connect( self.on_document_spam_attempt_save, sender=DocumentSpamAttempt, dispatch_uid="wiki.spam_attempt.post_save" )
class DJESConfig(AppConfig): name = 'djes' verbose_name = "DJ E.S." def ready(self): from .models import Indexable # Let's register all the Indexable models for model in apps.get_models(): # If it quacks... if issubclass(model, Indexable): meta = getattr(model, "_meta") if meta and not getattr(meta, "abstract"): indexable_registry.register(model) connections.configure(**settings.ES_CONNECTIONS)
#!/usr/bin/env python3 import sys import fileinput import json from elasticsearch_dsl import Document, field, InnerDoc from elasticsearch_dsl.connections import connections import logging logger = logging.getLogger('chibi_dl_elasticsearch.models.nhentai') connections.configure(default={'host': 'waifus', 'port': 80}) from elasticsearch_dsl import analyzer, tokenizer tag = analyzer( 'tag', tokenizer=tokenizer('trigram', 'nGram', min_gram=3, max_gram=3), filter=[ "lowercase", ], ) titles = analyzer( 'titles', tokenizer=tokenizer('trigram', 'nGram', min_gram=3, max_gram=5), filter=[ "lowercase", ], ) titles_space = analyzer(
######################################################## # Default connection from elasticsearch_dsl.connections import connections connections.create_connection(hosts=['localhost'], timeout=20) ############################################### # Multiple clusters connections.configure( default={'hosts': 'localhost'}, dev={ 'hosts': ['esdev1.example.com:9200'], 'sniff_on_start': True } ) # We can also add them later using aliases... connections.create_connection('qa', hosts=['esqa1.example.com'], sniff_on_start=True) connections.add_connection('qa', my_client) # When searching we can refer to a specific connection by using aliases s = Search(using='qa') ############################################### # The api is chainable s = Search().using(client).query('match', title='python')
# coding: utf-8 from elasticsearch_dsl.connections import connections ES_HOST = "10.10.0.247:9200" # ES_HOST = {"host": "192.168.10.190:9200"} connections.configure(default={"hosts": ES_HOST}) REDIS_HOST = "" REDIS_PASSWORD = "" # REDIS_HOST = "127.0.0.1" # REDIS_PASSWORD = "" ES_INDEX = "mxx_zjac_t2" ES_DOC_TYPE = "doc"
def configure(): connections_definitions = getattr(settings, 'FLEXY_CONNECTIONS', {}) connections.configure(**connections_definitions)
def ready(self): autodiscover_modules('documents') connections.configure(**settings.ELASTICSEARCH_DSL)
def ready(self): from .settings import DATABASES from elasticsearch_dsl.connections import connections connections.configure(**DATABASES)
from elasticsearch_dsl.query import Q from elasticsearch_dsl.connections import connections from .base import BaseFileManager logger = logging.getLogger(__name__) try: es_settings = getattr(settings, 'ELASTIC_SEARCH', {}) published_index = es_settings['published_index'] cluster = es_settings['cluster'] hosts = cluster['hosts'] connections.configure( default={ 'hosts': hosts, 'sniff_on_start': True, 'sniff_on_connection_fail': True, 'sniffer_timeout': 60, 'retry_on_timeout': True, 'timeout:': 20, }) except KeyError as e: logger.exception('ELASTIC_SEARCH missing %s' % e) class PublicProjectIndexed(DocType): class Meta: index = published_index doc_type = 'project' class PublicExperimentIndexed(DocType): class Meta: index = published_index
from elasticsearch_dsl.connections import connections from elasticsearch_dsl import DocType, Text, Date, Search from elasticsearch.helpers import bulk from elasticsearch import Elasticsearch from . import models connections.configure(default={'hosts': ['elk:9200'], 'sniff_on_start': True}) connections.create_connection(hosts='elk:9200') def bulk_indexing(): BlogPostIndex.init() es = Elasticsearch(['elk:9200']) bulk(client=es, actions=(b.indexing() for b in models.BlogPost.objects.all().iterator())) class BlogPostIndex(DocType): author = Text() posted_date = Date() title = Text() text = Text() class Meta: index = 'blogpost-index' def search(author): s = Search().filter('term', author=author) response = s.execute()
#使用该方法不能指定_id from elasticsearch import Elasticsearch from sqlalchemy import create_engine from elasticsearch_dsl import Document, Text, Integer, Keyword, Long, Boolean from elasticsearch_dsl.connections import connections connections.configure(default={'hosts': '192.168.59.129:9200'}) class TeacherType(Document): tno = Long(required=True) tname = Keyword() tsex = Keyword() tbirthday = Keyword() prof = Keyword() depart = Text(analyzer="ik_max_word") class Index(): name = 'teacher' # doc_type = 'teacher' class Mysql_Data_to_Es(): def __init__(self): self.db = create_engine( "mysql+pymysql://root:[email protected]:3306/sql_test", encoding='utf8') def get_mysql_date(self): resultProxy = self.db.execute('select * from teacher') data = resultProxy.fetchall()
except: print("ERROR: Did you remember to generate config files with credstmpl?", file=sys.stderr) print("Check out credstmpl at https://github.com/qadium/credstmpl", file=sys.stderr) print("you'll need to run `credstmpl filename.extension.j2`", file=sys.stderr) def es_config(**kwargs): return dict( verify_certs=False, use_ssl=False, request_timeout=160, timeout=160, **kwargs ) def get_config(): env = os.getenv('PRODUCTION', None) cfg = load_config('production') if env else load_config('develop') cfg.update(load_config("common")) logger.debug("Using {env} environment.".format(env=env)) return cfg cfg = get_config() connections.configure( cdr=es_config(**cfg['cdr_elastic_search']), local=es_config(hosts='localhost', index='factor_state2016'), )
from django.conf import settings from django.core.exceptions import ImproperlyConfigured from elasticsearch_dsl.connections import connections default_app_config = 'search.apps.SearchAppConfig' SEARCH_CONFIG = getattr(settings, 'SEARCH', None) if not SEARCH_CONFIG: raise ImproperlyConfigured('SEARCH settings dictionary must be provided') if 'default' not in SEARCH_CONFIG: raise ImproperlyConfigured("SEARCH settings must include 'default' settings dict, similar to DATABASES") if 'index' not in SEARCH_CONFIG['default']: raise ImproperlyConfigured("SEARCH settings must include a name for 'index' in 'default' settings dict") # Set up ES connection connection_settings = SEARCH_CONFIG['default'].get('connections') if not connection_settings: raise ImproperlyConfigured("SEARCH settings must include 'connections' dict in 'default'") connections.configure(default=connection_settings)
from elasticsearch.helpers import bulk from elasticsearch_dsl import document, field from elasticsearch_dsl.connections import connections from elasticsearch_dsl.mapping import Mapping from elasticsearch_dsl.search import Search from kuma.core.utils import chord_flow, chunked log = logging.getLogger('kuma.wiki.search') # Configure Elasticsearch connections for connection pooling. connections.configure( default={'hosts': settings.ES_URLS}, indexing={'hosts': settings.ES_URLS, 'timeout': settings.ES_INDEXING_TIMEOUT}, ) class WikiDocumentType(document.DocType): excerpt_fields = ['summary', 'content'] exclude_slugs = ['Talk:', 'User:'******'User_talk:', 'Template_talk:', 'Project_talk:'] boost = field.Float(null_value=1.0) content = field.String(analyzer='kuma_content', term_vector='with_positions_offsets') css_classnames = field.String(analyzer='case_insensitive_keyword') html_attributes = field.String(analyzer='case_insensitive_keyword') id = field.Long()
def ready(self): # Set up Elasticsearch connections based on settings urls = getattr(settings, 'ES_URLS', DEFAULT_URLS) timeout = getattr(settings, 'ES_TIMEOUT', DEFAULT_TIMEOUT) connections.configure(default={'hosts': urls, 'timeout': timeout})
pass def normalize_alphabet(rec): rec["term"] = normalize_charset(rec["term"]) return rec if __name__ == '__main__': if len(sys.argv) < 2: raise Exception("Input file argument is not specified") input_fname = sys.argv[1] if not os.path.exists(input_fname): raise Exception("Input file doesn't exist") connections.configure(**ELASTICSEARCH_CONNECTIONS) Index(NameVariant._doc_type.index).delete(ignore=404) NameVariant.init() accum = [] total_lines = 0 with open(input_fname, encoding="utf-8") as input_fp: for i, line in enumerate(input_fp): accum.append(loads(line)) if len(accum) >= 10000: total_lines += len(accum) bulk_load(map(normalize_alphabet, accum)) accum = [] print("Loaded: %s items" % total_lines) bulk_load(accum)
def createConnection(host, timeout): connections.configure( default={'hosts': host, 'timeout': timeout}, ) es = Elasticsearch() return es
MIDDLEWARE_CLASSES = ( 'django.middleware.common.CommonMiddleware', ) INSTALLED_APPS = ( 'django.contrib.auth', 'django.contrib.contenttypes', 'seeker', 'core', ) ROOT_URLCONF = 'urls' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:', } } LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True SEEKER_INDEX = 'seeker-tests' from elasticsearch_dsl.connections import connections connections.configure(default={'hosts': 'localhost'})
def ready(self): connections.configure(**settings.ELASTICSEARCH_DSL) autodiscover_modules("metrics")
def ready(self): connections.configure(**settings.ES_CONNECTIONS)