def get_es_client(hosts=None, headers=None):
global global_client
if not hosts:
if global_client is None:
global_client = AsyncElasticsearch(hosts=es_hosts)
return global_client
else:
return AsyncElasticsearch(hosts=hosts, headers=headers)
def test_retry_will_work(port, server, event_loop):
client = AsyncElasticsearch(hosts=['not-an-es-host', 'localhost'],
port=port,
loop=event_loop,
randomize_hosts=False)
data = yield from client.info()
assert {'body': '', 'method': 'GET', 'params': {}, 'path': '/'} == data
client.transport.close()
def test_retry_will_work(port, server, event_loop):
client = AsyncElasticsearch(
hosts=['not-an-es-host', 'localhost'],
port=port,
loop=event_loop,
randomize_hosts=False)
data = yield from client.info()
assert {'body': '', 'method': 'GET', 'params': {}, 'path': '/'} == data
yield from client.transport.close()
async def test_get(write_client):
async_client = AsyncElasticsearch(hosts=['localhost'])
elasticsearch_repo = await Repository.get('elasticsearch-dsl-py',
using=async_client)
assert isinstance(elasticsearch_repo, Repository)
assert elasticsearch_repo.owner.name == 'elasticsearch'
assert datetime(2014, 3, 3) == elasticsearch_repo.created_at
def test_multiple_hosts_make_async_pool(server, event_loop, port):
client = AsyncElasticsearch(hosts=['localhost', 'localhost'],
port=port,
loop=event_loop)
assert isinstance(client.transport.connection_pool, AsyncConnectionPool)
assert len(client.transport.connection_pool.orig_connections) == 2
yield from client.transport.close()
def init_with_loop(self, loop):
self.redis = loop.run_until_complete(
aioredis.create_redis((options.redis_host, options.redis_port),
loop=loop))
self.cache = RedisCacheBackend(self.redis)
es_hosts = [x.strip() for x in options.es_hosts.split(',')]
self.es = AsyncElasticsearch(hosts=es_hosts, loop=loop)
async def test_mget(write_client):
async_client = AsyncElasticsearch(hosts=['localhost'])
commits = await Commit.mget(COMMIT_DOCS_WITH_MISSING, using=async_client)
assert commits[0] is None
assert commits[1]._id == '3ca6e1e73a071a705b4babd2f581c91a2a3e5037'
assert commits[2] is None
assert commits[3]._id == 'eb3e543323f189fd7b698e66295427204fff5755'
async def test_msearch(data_client):
async_client = AsyncElasticsearch(hosts=['localhost'])
s = Search(using=async_client).index('git')
ms = MultiSearch(using=async_client).index('git')
ms = ms.add(s).add(s)
r1, r2 = await ms.execute()
assert all([r1.success(), r2.success()])
def client(event_loop, port):
c = AsyncElasticsearch([{
'host': '127.0.0.1',
'port': port
}],
loop=event_loop)
yield c
c.transport.close()
def test_async_pool_is_closed_properly(server, event_loop, port):
client = AsyncElasticsearch(hosts=['localhost', 'localhost'],
port=port,
loop=event_loop)
assert isinstance(client.transport.connection_pool, AsyncConnectionPool)
yield from client.transport.close()
for conn in client.transport.connection_pool.orig_connections:
assert conn.session.closed
async def elasticsearch_async(request):
"""Elasticsearch client fixture."""
client = AsyncElasticsearch(hosts=os.environ["ES_URL"])
try:
yield client
finally:
await client.indices.delete(index="*")
await client.transport.close()
def __init__(self):
# TODO: Set up connection with elasticsearch
# TODO: Read and store product master into a dict
self.product_master = {}
self.prepare_master()
self.client = AsyncElasticsearch(hosts=['localhost'])
# Create index if it does not exist already
asyncio.ensure_future(self.client.indices.create(index='data-stream', ignore=400))
print('ready')
async def aiomigrate(solrhost, eshost, name, solrfq, solrid):
LOGGER.info(
'asyncio migrate from solr (%s) into elasticsearch (%s) index %s '
'with filter query (%s) and with id (%s)', solrhost, eshost, name,
solrfq, solrid)
async with aiohttp.ClientSession() as session:
await Solr2EsAsync(session, AsyncElasticsearch(hosts=[eshost]),
solrhost).migrate(name,
solr_filter_query=solrfq,
sort_field=solrid)
def client(event_loop, server, port):
logger = logging.getLogger('elasticsearch')
logger.setLevel(logging.DEBUG)
c = AsyncElasticsearch([{
'host': '127.0.0.1',
'port': port
}],
loop=event_loop)
yield c
event_loop.run_until_complete(c.transport.close())
async def wait_until_up(self) -> None:
while (True):
log.info("Waiting for Elasticsearch to be Up...")
try:
res = await self.httpClient.get('http://elasticsearch:9200')
self.es = AsyncElasticsearch([{'host': 'elasticsearch', 'port': 9200}])
log.info("Elasticsearch is up")
break
except Exception as e:
await asyncio.sleep(1)
async def aioresume_from_pgsql(pgsqldsn, eshost, name, translationmap, es_index_body):
LOGGER.info('asyncio resume from postgresql (dsn=%s) to elasticsearch (%s) index %s', pgsqldsn, eshost, name)
dsndict = dict((kvstr.split('=') for kvstr in pgsqldsn.split()))
psql_queue = await PostgresqlQueueAsync.create(await create_engine(**dsndict))
es_index_body_str = None if es_index_body is None else dumps(es_index_body)
elasticsearch = AsyncElasticsearch([eshost], AsyncTransport, timeout=60)
await Solr2EsAsync(None, elasticsearch, None).\
resume(psql_queue, name, es_index_body_str, translationmap)
await elasticsearch.transport.close()
async def setup():
global client
conn = AsyncElasticsearch(hosts=[config['es_host']])
client[os.getpid()] = conn
try:
await get_client().indices.get(index=INDEX_NAME)
except Exception as e:
if getattr(e, 'error', '') == 'index_not_found_exception':
await get_client().indices.create(index=INDEX_NAME, body=INDEX)
else:
raise
async def test_save(write_client):
async_client = AsyncElasticsearch(hosts=['localhost'])
elasticsearch_repo = await Repository.get('elasticsearch-dsl-py',
using=async_client)
elasticsearch_repo.new_field = 'testing-save'
v = elasticsearch_repo.meta.version
assert not (await elasticsearch_repo.save(using=async_client))
# assert version has been updated
assert elasticsearch_repo.meta.version == v + 1
def init(sanic, loop):
global es
global users
global library_documents
from elasticsearch_async import AsyncElasticsearch
from motor.motor_asyncio import AsyncIOMotorClient
mongo_uri = "mongodb://127.0.0.1:27017/"
es = AsyncElasticsearch(hosts=['localhost'])
users = AsyncIOMotorClient(mongo_uri)['highload']['users']
library_documents = \
AsyncIOMotorClient(mongo_uri)['highload']['library_documents']
def update_emotions(index_name, routing_id, emotion_id, pos_neg):
logger.debug("[update] %s, %s" %(emotion_id, pos_neg))
request_body = {
"script":{
"source": "ctx._source.put('emotion_type', params.emotion_type); ctx._source.put('upd_datetime', params.upd_datetime);",
"params" : {
"emotion_type" : pos_neg,
"upd_datetime" : str(dt.now().strftime('%Y-%m-%dT%H:%M:%S'))
}
}
}
#es = AsyncElasticsearch(hosts=['%s:%d'%(es_ip, es_port)])
es = AsyncElasticsearch(hosts=[es_ip])
request_params= {
'routing' : routing_id
}
r = yield from es.update(index_name, TYPE_DOC, emotion_id, body=request_body, params=request_params)
es.transport.close()
def load_clients() -> None:
connections_dir = path.abspath(path.join(path.dirname(path.abspath(__file__)), 'clusters'))
for connection in glob(connections_dir + '/*'):
if not connection.endswith('.json'):
raise RuntimeError(f'Connections dir {connections_dir} requires json files')
info = json.load(open(connection))
if 'params' not in info:
raise RuntimeError(f'Missing params key in {connection}')
name = info.get('name')
if name is None:
name = connection.split('/')[-1].replace('.json', '')
info['client'] = AsyncElasticsearch(**ChainMap(info['params'], defaults))
clusters[name] = info
def test_sniff_on_start_sniffs(server, event_loop, port, sniff_data):
server.register_response('/_nodes/_all/clear', sniff_data)
client = AsyncElasticsearch(port=port, sniff_on_start=True, loop=event_loop)
# sniff has been called in the background
assert client.transport.sniffing_task is not None
yield from client.transport.sniffing_task
assert [('GET', '/_nodes/_all/clear', '', {})] == server.calls
connections = client.transport.connection_pool.connections
assert 1 == len(connections)
assert 'http://node1:9200' == connections[0].host
async def test(ip):
s = None
try:
client = AsyncElasticsearch(hosts=[ip])
# s = await client.cluster.health()
s = await client.cluster.stats()
found.append({
'ip': ip,
'versions': s['nodes']['versions'],
'stats': s
})
except Exception:
pass
if s:
print(ip)
def _init_client(self) -> Elasticsearch:
"""
Initialises the correct Elasticsearch client for the Sanic app
:return:
"""
es_host = self.elasticsearch_host
if self.elasticsearch_async_enabled:
from elasticsearch_async import AsyncElasticsearch
client = AsyncElasticsearch(es_host,
loop=self.loop,
timeout=self.elasticsearch_timeout)
else:
client = Elasticsearch(es_host, timeout=self.elasticsearch_timeout)
return client
async def before_server_start(app, loop):
conf = app.config
app.rdb = await aioredis.create_redis_pool(
(conf.REDIS_HOST, conf.REDIS_PORT),
db=conf.REDIS_DB,
encoding='utf8',
loop=loop)
app.es = AsyncElasticsearch(hosts=conf.ES_HOST)
token = json.loads(await app.rdb.get('token') or '{}')
if not token or token['expiration'] < time.time() + 3600 * 24 * 7:
token = await fetch_aip_token(app.config)
await app.rdb.set('token', json.dumps(token))
app.token = token['access_token']
app.last_train_timestamp = 0
async def test_update(write_client):
async_client = AsyncElasticsearch(hosts=['localhost'])
elasticsearch_repo = await Repository.get('elasticsearch-dsl-py',
using=async_client)
v = elasticsearch_repo.meta.version
await elasticsearch_repo.update(owner={'new_name': 'elastic_async'},
new_field='testing-update',
using=async_client)
assert 'elastic_async' == elasticsearch_repo.owner.new_name
assert 'testing-update' == elasticsearch_repo.new_field
# assert version has been updated
assert elasticsearch_repo.meta.version == v + 1
new_version = await Repository.get('elasticsearch-dsl-py',
using=async_client)
assert 'testing-update' == new_version.new_field
assert 'elastic_async' == new_version.owner.new_name
assert 'elasticsearch' == new_version.owner.name
def test_streaming_bulk(self):
client = AsyncElasticsearch(
hosts=config.CONFIG.ELASTICSEARCH.get('hosts'),
timeout=120,
retry_on_timeout=True,
sniff_on_start=False,
sniff_on_connection_fail=True,
sniffer_timeout=60,
max_retries=3,
serializer=BSONSerializer())
actions = [
{
'_op_type': 'index',
'_index': 'rts_test',
'_type': 'rt',
'_id': 1,
'now': datetime.datetime.now()
},
{
'_op_type': 'index',
'_index': 'rts_test',
'_type': 'rt',
'_id': 1,
'now': datetime.datetime.now()
},
]
future = async_helpers.bulk(client=client,
actions=actions,
max_retries=3,
initial_backoff=0.1,
max_backoff=1)
loop = asyncio.get_event_loop()
succeed, failed = loop.run_until_complete(future)
print('succeed:', len(succeed), '\nfailed:', len(failed))
print(succeed, failed)
loop.run_until_complete(asyncio.wait(asyncio.Task.all_tasks()))
loop.close()
def __init__(self, hosts):
self._client = AsyncElasticsearch(hosts=hosts, maxsize=40)
BULK_SIZE = 10
TOTAL_DOCS = 2000000
INDEX_NAME = 'bench'
DOC_TYPE = '_doc'
async def bench(es):
nb_indexed = 0
for i in range(1, TOTAL_DOCS, BULK_SIZE):
bulk_content = str()
for bulk_index in range(i, i + BULK_SIZE):
es_action = dumps({'index': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': i}}) + '\n' + \
dumps({'field': 'my content %d' % bulk_index}) + '\n'
bulk_content += es_action
await es.bulk(bulk_content, INDEX_NAME, DOC_TYPE, refresh=True)
nb_indexed += BULK_SIZE
if nb_indexed % 1000 == 0:
LOGGER.info('indexed %s docs of %s (%.2f %% done)', nb_indexed,
TOTAL_DOCS, (100 * nb_indexed) / TOTAL_DOCS)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
elasticsearch = AsyncElasticsearch(['elasticsearch'],
AsyncTransport,
timeout=60)
try:
loop.run_until_complete(bench(elasticsearch))
finally:
loop.run_until_complete(elasticsearch.transport.close())
async def main():
client = AsyncElasticsearch(hosts=conf['elasticsearch']['hosts'])
for index in conf['elasticsearch']['indices']:
await client.indices.create(index, body=app.storage.es.body.get(index))
await client.transport.close()
async def _process_bulk_chunk(client: AsyncElasticsearch,
bulk_actions,
bulk_data,
max_retries,
initial_backoff,
max_backoff,
**kwargs):
"""
Send a bulk request to elasticsearch and process the output, it will retry when exception raised.
"""
attempted = 0
succeed, failed = [], []
while attempted <= max_retries:
# send the actual request
future = client.bulk('\n'.join(bulk_actions) + '\n', **kwargs)
attempted += 1
# if raise on error is set, we need to collect errors per chunk before raising them
try:
result = await future
except TransportError as e:
logger.warning('[Elasticsearch] %r', e)
if type(e) in NO_RETRY_EXCEPTIONS or attempted > max_retries:
# if we are not propagating, mark all actions in current chunk as failed
err_message = str(e)
for data in bulk_data:
# collect all the information about failed actions
op_type, action = data[0].copy().popitem()
info = {"error": err_message, "status": e.status_code, "create_time": util.utc_now()}
if op_type != 'delete':
info['data'] = data[1]
info['action'] = action
failed.append(info)
except Exception as e:
logger.warning('[AsyncHelper] %r', e)
if attempted > max_retries:
# if we are not propagating, mark all actions in current chunk as failed
err_message = str(e)
for data in bulk_data:
# collect all the information about failed actions
op_type, action = data[0].copy().popitem()
info = {"error": err_message, "status": 500, "create_time": util.utc_now(), "action": action}
# if op_type != 'delete':
# info['data'] = data[1]
failed.append(info)
else:
to_retry, to_retry_data = [], []
# go through request-response pairs and detect failures
for (action, data), (ok, info) in zip(bulk_data, _chunk_result(bulk_data, result)):
op, info = info.popitem()
if not ok and info.get('status') != 404:
if attempted < max_retries:
to_retry.append(client.transport.serializer.dumps(action))
if data:
to_retry.append(client.transport.serializer.dumps(data))
to_retry_data.append((action, data))
else:
info = {
'error': str(info.get('error')),
'status': info.get('status'),
'action': action,
# 'data': data,
'create_time': util.utc_now()
}
failed.append(info)
else:
# succeed or max retry
succeed.append(info)
# retry only subset of documents that didn't succeed
if attempted < max_retries:
bulk_actions, bulk_data = to_retry, to_retry_data
if not to_retry:
# all success, no need to retry
break
finally:
delay = min(max_backoff, initial_backoff * 2 ** (attempted - 1))
await asyncio.sleep(delay)
if attempted <= max_retries:
logger.debug('Elasticsearch bulk request retry')
return succeed, failed
