def get_es_client(hosts=None, headers=None): global global_client if not hosts: if global_client is None: global_client = AsyncElasticsearch(hosts=es_hosts) return global_client else: return AsyncElasticsearch(hosts=hosts, headers=headers)
def test_retry_will_work(port, server, event_loop): client = AsyncElasticsearch(hosts=['not-an-es-host', 'localhost'], port=port, loop=event_loop, randomize_hosts=False) data = yield from client.info() assert {'body': '', 'method': 'GET', 'params': {}, 'path': '/'} == data client.transport.close()
def test_retry_will_work(port, server, event_loop): client = AsyncElasticsearch( hosts=['not-an-es-host', 'localhost'], port=port, loop=event_loop, randomize_hosts=False) data = yield from client.info() assert {'body': '', 'method': 'GET', 'params': {}, 'path': '/'} == data yield from client.transport.close()
async def test_get(write_client): async_client = AsyncElasticsearch(hosts=['localhost']) elasticsearch_repo = await Repository.get('elasticsearch-dsl-py', using=async_client) assert isinstance(elasticsearch_repo, Repository) assert elasticsearch_repo.owner.name == 'elasticsearch' assert datetime(2014, 3, 3) == elasticsearch_repo.created_at
def test_multiple_hosts_make_async_pool(server, event_loop, port): client = AsyncElasticsearch(hosts=['localhost', 'localhost'], port=port, loop=event_loop) assert isinstance(client.transport.connection_pool, AsyncConnectionPool) assert len(client.transport.connection_pool.orig_connections) == 2 yield from client.transport.close()
def init_with_loop(self, loop): self.redis = loop.run_until_complete( aioredis.create_redis((options.redis_host, options.redis_port), loop=loop)) self.cache = RedisCacheBackend(self.redis) es_hosts = [x.strip() for x in options.es_hosts.split(',')] self.es = AsyncElasticsearch(hosts=es_hosts, loop=loop)
async def test_mget(write_client): async_client = AsyncElasticsearch(hosts=['localhost']) commits = await Commit.mget(COMMIT_DOCS_WITH_MISSING, using=async_client) assert commits[0] is None assert commits[1]._id == '3ca6e1e73a071a705b4babd2f581c91a2a3e5037' assert commits[2] is None assert commits[3]._id == 'eb3e543323f189fd7b698e66295427204fff5755'
async def test_msearch(data_client): async_client = AsyncElasticsearch(hosts=['localhost']) s = Search(using=async_client).index('git') ms = MultiSearch(using=async_client).index('git') ms = ms.add(s).add(s) r1, r2 = await ms.execute() assert all([r1.success(), r2.success()])
def client(event_loop, port): c = AsyncElasticsearch([{ 'host': '127.0.0.1', 'port': port }], loop=event_loop) yield c c.transport.close()
def test_async_pool_is_closed_properly(server, event_loop, port): client = AsyncElasticsearch(hosts=['localhost', 'localhost'], port=port, loop=event_loop) assert isinstance(client.transport.connection_pool, AsyncConnectionPool) yield from client.transport.close() for conn in client.transport.connection_pool.orig_connections: assert conn.session.closed
async def elasticsearch_async(request): """Elasticsearch client fixture.""" client = AsyncElasticsearch(hosts=os.environ["ES_URL"]) try: yield client finally: await client.indices.delete(index="*") await client.transport.close()
def __init__(self): # TODO: Set up connection with elasticsearch # TODO: Read and store product master into a dict self.product_master = {} self.prepare_master() self.client = AsyncElasticsearch(hosts=['localhost']) # Create index if it does not exist already asyncio.ensure_future(self.client.indices.create(index='data-stream', ignore=400)) print('ready')
async def aiomigrate(solrhost, eshost, name, solrfq, solrid): LOGGER.info( 'asyncio migrate from solr (%s) into elasticsearch (%s) index %s ' 'with filter query (%s) and with id (%s)', solrhost, eshost, name, solrfq, solrid) async with aiohttp.ClientSession() as session: await Solr2EsAsync(session, AsyncElasticsearch(hosts=[eshost]), solrhost).migrate(name, solr_filter_query=solrfq, sort_field=solrid)
def client(event_loop, server, port): logger = logging.getLogger('elasticsearch') logger.setLevel(logging.DEBUG) c = AsyncElasticsearch([{ 'host': '127.0.0.1', 'port': port }], loop=event_loop) yield c event_loop.run_until_complete(c.transport.close())
async def wait_until_up(self) -> None: while (True): log.info("Waiting for Elasticsearch to be Up...") try: res = await self.httpClient.get('http://elasticsearch:9200') self.es = AsyncElasticsearch([{'host': 'elasticsearch', 'port': 9200}]) log.info("Elasticsearch is up") break except Exception as e: await asyncio.sleep(1)
async def aioresume_from_pgsql(pgsqldsn, eshost, name, translationmap, es_index_body): LOGGER.info('asyncio resume from postgresql (dsn=%s) to elasticsearch (%s) index %s', pgsqldsn, eshost, name) dsndict = dict((kvstr.split('=') for kvstr in pgsqldsn.split())) psql_queue = await PostgresqlQueueAsync.create(await create_engine(**dsndict)) es_index_body_str = None if es_index_body is None else dumps(es_index_body) elasticsearch = AsyncElasticsearch([eshost], AsyncTransport, timeout=60) await Solr2EsAsync(None, elasticsearch, None).\ resume(psql_queue, name, es_index_body_str, translationmap) await elasticsearch.transport.close()
async def setup(): global client conn = AsyncElasticsearch(hosts=[config['es_host']]) client[os.getpid()] = conn try: await get_client().indices.get(index=INDEX_NAME) except Exception as e: if getattr(e, 'error', '') == 'index_not_found_exception': await get_client().indices.create(index=INDEX_NAME, body=INDEX) else: raise
async def test_save(write_client): async_client = AsyncElasticsearch(hosts=['localhost']) elasticsearch_repo = await Repository.get('elasticsearch-dsl-py', using=async_client) elasticsearch_repo.new_field = 'testing-save' v = elasticsearch_repo.meta.version assert not (await elasticsearch_repo.save(using=async_client)) # assert version has been updated assert elasticsearch_repo.meta.version == v + 1
def init(sanic, loop): global es global users global library_documents from elasticsearch_async import AsyncElasticsearch from motor.motor_asyncio import AsyncIOMotorClient mongo_uri = "mongodb://127.0.0.1:27017/" es = AsyncElasticsearch(hosts=['localhost']) users = AsyncIOMotorClient(mongo_uri)['highload']['users'] library_documents = \ AsyncIOMotorClient(mongo_uri)['highload']['library_documents']
def update_emotions(index_name, routing_id, emotion_id, pos_neg): logger.debug("[update] %s, %s" %(emotion_id, pos_neg)) request_body = { "script":{ "source": "ctx._source.put('emotion_type', params.emotion_type); ctx._source.put('upd_datetime', params.upd_datetime);", "params" : { "emotion_type" : pos_neg, "upd_datetime" : str(dt.now().strftime('%Y-%m-%dT%H:%M:%S')) } } } #es = AsyncElasticsearch(hosts=['%s:%d'%(es_ip, es_port)]) es = AsyncElasticsearch(hosts=[es_ip]) request_params= { 'routing' : routing_id } r = yield from es.update(index_name, TYPE_DOC, emotion_id, body=request_body, params=request_params) es.transport.close()
def load_clients() -> None: connections_dir = path.abspath(path.join(path.dirname(path.abspath(__file__)), 'clusters')) for connection in glob(connections_dir + '/*'): if not connection.endswith('.json'): raise RuntimeError(f'Connections dir {connections_dir} requires json files') info = json.load(open(connection)) if 'params' not in info: raise RuntimeError(f'Missing params key in {connection}') name = info.get('name') if name is None: name = connection.split('/')[-1].replace('.json', '') info['client'] = AsyncElasticsearch(**ChainMap(info['params'], defaults)) clusters[name] = info
def test_sniff_on_start_sniffs(server, event_loop, port, sniff_data): server.register_response('/_nodes/_all/clear', sniff_data) client = AsyncElasticsearch(port=port, sniff_on_start=True, loop=event_loop) # sniff has been called in the background assert client.transport.sniffing_task is not None yield from client.transport.sniffing_task assert [('GET', '/_nodes/_all/clear', '', {})] == server.calls connections = client.transport.connection_pool.connections assert 1 == len(connections) assert 'http://node1:9200' == connections[0].host
async def test(ip): s = None try: client = AsyncElasticsearch(hosts=[ip]) # s = await client.cluster.health() s = await client.cluster.stats() found.append({ 'ip': ip, 'versions': s['nodes']['versions'], 'stats': s }) except Exception: pass if s: print(ip)
def _init_client(self) -> Elasticsearch: """ Initialises the correct Elasticsearch client for the Sanic app :return: """ es_host = self.elasticsearch_host if self.elasticsearch_async_enabled: from elasticsearch_async import AsyncElasticsearch client = AsyncElasticsearch(es_host, loop=self.loop, timeout=self.elasticsearch_timeout) else: client = Elasticsearch(es_host, timeout=self.elasticsearch_timeout) return client
async def before_server_start(app, loop): conf = app.config app.rdb = await aioredis.create_redis_pool( (conf.REDIS_HOST, conf.REDIS_PORT), db=conf.REDIS_DB, encoding='utf8', loop=loop) app.es = AsyncElasticsearch(hosts=conf.ES_HOST) token = json.loads(await app.rdb.get('token') or '{}') if not token or token['expiration'] < time.time() + 3600 * 24 * 7: token = await fetch_aip_token(app.config) await app.rdb.set('token', json.dumps(token)) app.token = token['access_token'] app.last_train_timestamp = 0
async def test_update(write_client): async_client = AsyncElasticsearch(hosts=['localhost']) elasticsearch_repo = await Repository.get('elasticsearch-dsl-py', using=async_client) v = elasticsearch_repo.meta.version await elasticsearch_repo.update(owner={'new_name': 'elastic_async'}, new_field='testing-update', using=async_client) assert 'elastic_async' == elasticsearch_repo.owner.new_name assert 'testing-update' == elasticsearch_repo.new_field # assert version has been updated assert elasticsearch_repo.meta.version == v + 1 new_version = await Repository.get('elasticsearch-dsl-py', using=async_client) assert 'testing-update' == new_version.new_field assert 'elastic_async' == new_version.owner.new_name assert 'elasticsearch' == new_version.owner.name
def test_streaming_bulk(self): client = AsyncElasticsearch( hosts=config.CONFIG.ELASTICSEARCH.get('hosts'), timeout=120, retry_on_timeout=True, sniff_on_start=False, sniff_on_connection_fail=True, sniffer_timeout=60, max_retries=3, serializer=BSONSerializer()) actions = [ { '_op_type': 'index', '_index': 'rts_test', '_type': 'rt', '_id': 1, 'now': datetime.datetime.now() }, { '_op_type': 'index', '_index': 'rts_test', '_type': 'rt', '_id': 1, 'now': datetime.datetime.now() }, ] future = async_helpers.bulk(client=client, actions=actions, max_retries=3, initial_backoff=0.1, max_backoff=1) loop = asyncio.get_event_loop() succeed, failed = loop.run_until_complete(future) print('succeed:', len(succeed), '\nfailed:', len(failed)) print(succeed, failed) loop.run_until_complete(asyncio.wait(asyncio.Task.all_tasks())) loop.close()
def __init__(self, hosts): self._client = AsyncElasticsearch(hosts=hosts, maxsize=40)
BULK_SIZE = 10 TOTAL_DOCS = 2000000 INDEX_NAME = 'bench' DOC_TYPE = '_doc' async def bench(es): nb_indexed = 0 for i in range(1, TOTAL_DOCS, BULK_SIZE): bulk_content = str() for bulk_index in range(i, i + BULK_SIZE): es_action = dumps({'index': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': i}}) + '\n' + \ dumps({'field': 'my content %d' % bulk_index}) + '\n' bulk_content += es_action await es.bulk(bulk_content, INDEX_NAME, DOC_TYPE, refresh=True) nb_indexed += BULK_SIZE if nb_indexed % 1000 == 0: LOGGER.info('indexed %s docs of %s (%.2f %% done)', nb_indexed, TOTAL_DOCS, (100 * nb_indexed) / TOTAL_DOCS) if __name__ == '__main__': loop = asyncio.get_event_loop() elasticsearch = AsyncElasticsearch(['elasticsearch'], AsyncTransport, timeout=60) try: loop.run_until_complete(bench(elasticsearch)) finally: loop.run_until_complete(elasticsearch.transport.close())
async def main(): client = AsyncElasticsearch(hosts=conf['elasticsearch']['hosts']) for index in conf['elasticsearch']['indices']: await client.indices.create(index, body=app.storage.es.body.get(index)) await client.transport.close()
async def _process_bulk_chunk(client: AsyncElasticsearch, bulk_actions, bulk_data, max_retries, initial_backoff, max_backoff, **kwargs): """ Send a bulk request to elasticsearch and process the output, it will retry when exception raised. """ attempted = 0 succeed, failed = [], [] while attempted <= max_retries: # send the actual request future = client.bulk('\n'.join(bulk_actions) + '\n', **kwargs) attempted += 1 # if raise on error is set, we need to collect errors per chunk before raising them try: result = await future except TransportError as e: logger.warning('[Elasticsearch] %r', e) if type(e) in NO_RETRY_EXCEPTIONS or attempted > max_retries: # if we are not propagating, mark all actions in current chunk as failed err_message = str(e) for data in bulk_data: # collect all the information about failed actions op_type, action = data[0].copy().popitem() info = {"error": err_message, "status": e.status_code, "create_time": util.utc_now()} if op_type != 'delete': info['data'] = data[1] info['action'] = action failed.append(info) except Exception as e: logger.warning('[AsyncHelper] %r', e) if attempted > max_retries: # if we are not propagating, mark all actions in current chunk as failed err_message = str(e) for data in bulk_data: # collect all the information about failed actions op_type, action = data[0].copy().popitem() info = {"error": err_message, "status": 500, "create_time": util.utc_now(), "action": action} # if op_type != 'delete': # info['data'] = data[1] failed.append(info) else: to_retry, to_retry_data = [], [] # go through request-response pairs and detect failures for (action, data), (ok, info) in zip(bulk_data, _chunk_result(bulk_data, result)): op, info = info.popitem() if not ok and info.get('status') != 404: if attempted < max_retries: to_retry.append(client.transport.serializer.dumps(action)) if data: to_retry.append(client.transport.serializer.dumps(data)) to_retry_data.append((action, data)) else: info = { 'error': str(info.get('error')), 'status': info.get('status'), 'action': action, # 'data': data, 'create_time': util.utc_now() } failed.append(info) else: # succeed or max retry succeed.append(info) # retry only subset of documents that didn't succeed if attempted < max_retries: bulk_actions, bulk_data = to_retry, to_retry_data if not to_retry: # all success, no need to retry break finally: delay = min(max_backoff, initial_backoff * 2 ** (attempted - 1)) await asyncio.sleep(delay) if attempted <= max_retries: logger.debug('Elasticsearch bulk request retry') return succeed, failed