async def start(self) -> bool:
if self.sync_client:
return False
hosts = [{'host': self._elastic_host, 'port': self._elastic_port}]
self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout)
self.search_client = AsyncElasticsearch(hosts,
timeout=self.search_timeout)
while True:
try:
await self.sync_client.cluster.health(wait_for_status='yellow')
break
except ConnectionError:
self.logger.warning(
"Failed to connect to Elasticsearch. Waiting for it!")
await asyncio.sleep(1)
res = await self.sync_client.indices.create(self.index,
INDEX_DEFAULT_SETTINGS,
ignore=400)
acked = res.get('acknowledged', False)
if acked:
await self.set_index_version(self.VERSION)
return acked
index_version = await self.get_index_version()
if index_version != self.VERSION:
self.logger.error(
"es search index has an incompatible version: %s vs %s",
index_version, self.VERSION)
raise IndexVersionMismatch(index_version, self.VERSION)
await self.sync_client.indices.refresh(self.index)
return acked
def _create_es_client(self, sync=False):
"""Constructs an instance of the Elasticsearch client"""
from elasticsearch import AsyncElasticsearch, Elasticsearch
use_basic_auth = self._username is not None and self._password is not None
serializer = get_serializer()
if use_basic_auth:
auth = (self._username, self._password)
return Elasticsearch(
[self._url],
http_auth=auth,
serializer=serializer,
verify_certs=self._verify_certs,
ca_certs=self._ca_certs,
timeout=self._timeout) if sync else AsyncElasticsearch(
[self._url],
http_auth=auth,
serializer=serializer,
verify_certs=self._verify_certs,
ca_certs=self._ca_certs,
timeout=self._timeout)
return Elasticsearch(
[self._url],
serializer=serializer,
verify_certs=self._verify_certs,
ca_certs=self._ca_certs,
timeout=self._timeout) if sync else AsyncElasticsearch(
[self._url],
serializer=serializer,
verify_certs=self._verify_certs,
ca_certs=self._ca_certs,
timeout=self._timeout)
def get_es_client(hosts=None, headers=None):
global global_client
if not hosts:
if global_client is None:
global_client = AsyncElasticsearch(hosts=es_hosts, headers=headers, http_auth=http_auth)
return global_client
else:
return AsyncElasticsearch(hosts=hosts, headers=headers, http_auth=http_auth)
def pre_index(self, *args, mode, **kwargs):
client = AsyncElasticsearch(**self.es_client_args)
try:
if mode in ("index", None):
# index MUST NOT exist
# ----------------------
if (yield from client.indices.exists(self.es_index_name)):
msg = (
"Index '%s' already exists, (use mode='purge' to "
"auto-delete it or mode='resume' to add more documents)"
)
raise IndexerException(msg % self.es_index_name)
elif mode in ("resume", "merge"):
# index MUST exist
# ------------------
if not (yield from client.indices.exists(self.es_index_name)):
raise IndexerException("'%s' does not exist." %
self.es_index_name)
self.logger.info(("Exists", self.es_index_name))
return # skip index creation
elif mode == "purge":
# index MAY exist
# -----------------
response = yield from client.indices.delete(
self.es_index_name, ignore_unavailable=True)
self.logger.info(("Deleted", self.es_index_name, response))
else:
raise ValueError("Invalid mode: %s" % mode)
response = yield from client.indices.create(
self.es_index_name,
body={
"settings": (yield from
self.es_index_settings.finalize(client)),
"mappings": (yield from
self.es_index_mappings.finalize(client))
})
self.logger.info(("Created", self.es_index_name, response))
return {
'__REPLACE__': True,
'host':
self.es_client_args.get('hosts'), # for frontend display
'environment': self.env_name, # used in snapshot module.
}
finally:
yield from client.close()
def __init__(self):
self.cfg = config.get_config()
self.url = self.cfg.get('elasticsearch.url')
self.indice = self.cfg.get('elasticsearch.indice')
if self.cfg.is_set('elasticsearch.username') and \
self.cfg.is_set('elasticsearch.password'):
self.es = AsyncElasticsearch(
self.url,
http_auth=(self.cfg.get('elasticsearch.username'),
self.cfg.get('elasticsearch.password')))
else:
self.es = AsyncElasticsearch(self.url)
async def start(self):
if self.sync_client:
return
self.sync_client = AsyncElasticsearch(timeout=self.sync_timeout)
self.search_client = AsyncElasticsearch(timeout=self.search_timeout)
while True:
try:
await self.sync_client.cluster.health(wait_for_status='yellow')
break
except ConnectionError:
self.logger.warning("Failed to connect to Elasticsearch. Waiting for it!")
await asyncio.sleep(1)
res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
return res.get('acknowledged', False)
async def connect_to_es():
settings = config.get_settings()
hosts = settings.es_hosts
opts = {}
if (settings.es_user is not None and settings.es_password is not None):
opts["http_auth"] = (settings.es_user, settings.es_password)
db.client = AsyncElasticsearch(hosts, **opts)
async def consume(producer):
es = AsyncElasticsearch()
try:
await async_bulk(es, producer, request_timeout=120)
await es.indices.refresh(index=INDEX)
finally:
await es.close()
async def async_elasticsearch(request):
"""AsyncElasticsearch client fixture."""
client = AsyncElasticsearch(hosts=os.environ["ES_URL"])
try:
yield client
finally:
await client.indices.delete(index="*")
await client.close()
def _validate_mapping():
client = AsyncElasticsearch(**indexer.es_client_args)
index_name = ("hub_tmp_%s" % get_random_string()).lower()
try:
return (yield from client.indices.create(
index_name,
body={
"settings":
(yield from
indexer.es_index_settings.finalize(client)),
"mappings":
(yield from indexer.es_index_mappings.finalize(client))
}))
finally:
yield from client.indices.delete(index_name,
ignore_unavailable=True)
yield from client.close()
def __init__(self, KBPATH: str):
self.es = AsyncElasticsearch([{
"host": "localhost",
"port": 9200
}],
timeout=30)
self.db = trident.Db(KBPATH)
self.loop = asyncio.get_event_loop()
self.cache: Dict[str, str] = {}
def init_es(
settings: Settings,
use_async: bool = True) -> Union[AsyncElasticsearch, Elasticsearch]:
"""Instantiate an elastic search client."""
if use_async:
return AsyncElasticsearch([settings.ES_HOST])
else:
client = Elasticsearch([settings.ES_HOST])
add_connection("default", client)
return client
def __init__(self, type: str, config: dict):
self.es = AsyncElasticsearch(config["elasticsearch"]["host"],
scheme=config["elasticsearch"]["scheme"],
port=config["elasticsearch"]["port"])
self.indexes_settings = config["elasticsearch"]["indexes"]["settings"]
self.type = type
self.mapping_file = config["app"]["mapping_file"].format(type=type)
now = datetime.now()
self.index_name = f"{type}_{now:%Y%m%d_%H%M%S}"
def __init__(self, *, auth, hosts=None, port=443, region=None):
self.client = AsyncElasticsearch(
http_auth=auth,
hosts=hosts or ['127.0.0.1'],
use_ssl=True,
verify_cert=True,
ssl_show_warn=False,
scheme='https',
port=port,
connection_class=AIOHttpConnection,
)
async def consume(producer, index_name):
env = Env(LBC)
logging.info("ES sync host: %s:%i", env.elastic_host, env.elastic_port)
es = AsyncElasticsearch([{
'host': env.elastic_host,
'port': env.elastic_port
}])
try:
await async_bulk(es, producer, request_timeout=120)
await es.indices.refresh(index=index_name)
finally:
await es.close()
async def startup():
redis.redis = await aioredis.create_redis_pool((settings.RECOMMENDATIONS_HOST, settings.RECOMMENDATIONS_PORT))
logger.info(f'CONNECTED TO REDIS SERVER {settings.RECOMMENDATIONS_HOST}:{settings.RECOMMENDATIONS_PORT}')
elastic.es = AsyncElasticsearch(
hosts=settings.ELASTIC_HOSTS,
use_ssl=True,
verify_certs=True,
http_auth=(settings.ELASTIC_USER, settings.ELASTIC_PASS),
ca_certs=settings.ELASTIC_CA_PATH)
logger.info(f'CONNECTED TO ELASTICSEARCH SERVER {settings.ELASTIC_HOSTS}')
async def get_lists(database: plugins.configuration.DBConfig) -> dict:
"""
:param database: a Pony Mail database configuration
:return: A dictionary of all mailing lists found, and whether they are considered
public or private
"""
lists = {}
client = AsyncElasticsearch([
{
"host": database.hostname,
"port": database.port,
"url_prefix": database.url_prefix or "",
"use_ssl": database.secure,
},
])
# Fetch aggregations of all public emails
s = Search(using=client,
index=database.db_prefix + "-mbox").query("match",
private=False)
s.aggs.bucket("per_list", "terms", field="list_raw")
res = await client.search(index=database.db_prefix + "-mbox",
body=s.to_dict(),
size=0)
for ml in res["aggregations"]["per_list"]["buckets"]:
list_name = ml["key"].strip("<>").replace(".", "@", 1)
lists[list_name] = {
"count": ml["doc_count"],
"private": False,
}
# Ditto, for private emails
s = Search(using=client,
index=database.db_prefix + "-mbox").query("match", private=True)
s.aggs.bucket("per_list", "terms", field="list_raw")
res = await client.search(index=database.db_prefix + "-mbox",
body=s.to_dict(),
size=0)
for ml in res["aggregations"]["per_list"]["buckets"]:
list_name = ml["key"].strip("<>").replace(".", "@", 1)
lists[list_name] = {
"count": ml["doc_count"],
"private": True,
}
await client.close()
return lists
async def startup():
# Подключаемся к базам при старте сервера
# Подключиться можем при работающем event-loop
# Поэтому логика подключения происходит в асинхронной функции
redis.redis = await aioredis.create_redis_pool(
(config.REDIS_HOST, config.REDIS_PORT),
password=config.REDIS_PASSWORD,
minsize=10,
maxsize=20)
elastic.es = AsyncElasticsearch(
hosts=[f'{config.ELASTIC_HOST}:{config.ELASTIC_PORT}'],
scheme=config.ELASTIC_SCHEME,
http_auth=(config.ELASTIC_USER, config.ELASTIC_PASSWORD))
async def get_connection() -> AsyncElasticsearch:
"""
Returns a connection to the Elasticsearch.
The connection is cached and reused. It is not
creating a new connection every time.
Required environment variables to create a connection:
- ES_HOST
- ES_PORT
- ES_USE_SSL
Optional parameters are ES_USERNAME and ES_PASSWORD which will add an http auth to the connection
constructor
"""
global _client
if _client:
return _client
if es_username:
client = AsyncElasticsearch(
hosts=[es_host],
port=int(es_port),
use_ssl=bool(strtobool(es_use_ssl)),
http_auth=(es_username, es_password),
verify_certs=True,
http_compress=True,
max_retries=5,
retry_on_timeout=True,
)
else:
client = AsyncElasticsearch(
hosts=[es_host],
port=int(es_port),
use_ssl=bool(strtobool(es_use_ssl)),
verify_certs=True,
http_compress=True,
max_retries=5,
retry_on_timeout=True,
)
_client = client
return client
def test_01():
import asyncio
from elasticsearch import AsyncElasticsearch
client = AsyncElasticsearch()
mappings = IndexMappings(DEFAULT_INDEX_MAPPINGS)
mappings["dynamic"] = True
async def finalize_mapping():
print(await mappings.finalize(client))
await client.close()
loop = asyncio.get_event_loop()
loop.run_until_complete(finalize_mapping())
async def startup():
redis.redis = await aioredis.create_redis_pool(
(config.REDIS_HOST, config.REDIS_PORT), minsize=10, maxsize=20)
elastic.es = AsyncElasticsearch(hosts=config.ELASTIC_HOSTS,
use_ssl=True,
verify_certs=True,
http_auth=(config.ELASTIC_USER,
config.ELASTIC_PASS),
ca_certs=config.ELASTIC_CA_PATH)
kafka.kafka_producer = AIOKafkaProducer(
bootstrap_servers=f'{config.BROKER_HOST}:{config.BROKER_PORT}',
key_serializer=str_to_bytes,
value_serializer=str_to_bytes)
await kafka.kafka_producer.start()
async def connect(self) -> None:
"""
It is used for initialize connections to elasticsearch instance(s)
"""
while True:
try:
self.client = AsyncElasticsearch(**self.client_options)
await self.client.info()
except elasticsearch.TransportError as e:
_logger.error(f"[{self.name}] Connection Error: {e}")
await asyncio.sleep(self.retry_delay_short)
else:
_logger.info(f"[{self.name}] Connected to "
f"{self.hosts}/{self.index}")
break
async def _enhance(conf):
conf = copy.deepcopy(conf)
for name, env in self.register.items():
async with AsyncElasticsearch(**env["args"]) as client:
try:
indices = await client.indices.get("*")
except elasticsearch.exceptions.ConnectionError:
... # keep the hard-coded place-holders info
else: # replace the index key with remote info
conf["env"][name]["index"] = [{
"index":
k,
"aliases":
list(v["aliases"].keys()),
} for k, v in indices.items()]
return conf
async def _update_meta(_meta):
env = self.register[indexer_env]
async with AsyncElasticsearch(**env["args"]) as client:
doc_type = None
if int((await
client.info())['version']['number'].split('.')[0]) < 7:
mappings = client.indices.get_mapping(index_name)
mappings = mappings[index_name]["mappings"]
doc_type = next(iter(mappings.keys()))
if _meta is None:
_id = build_name or index_name # best guess
build = get_src_build().find_one({"_id": _id})
_meta = (build or {}).get("_meta")
return await client.indices.put_mapping(body=dict(_meta=_meta),
index=index_name,
doc_type=doc_type)
async def elasticsearch_fixture(request):
"""Elasticsearch client fixture."""
process = request.getfixturevalue(process_fixture_name)
if not process.running():
process.start()
client = AsyncElasticsearch([{
'host': process.host,
'port': process.port
}])
async def drop_indexes():
await client.indices.delete(index='*')
await client.close()
def sync_drop_indexes():
asyncio.get_event_loop().run_until_complete(drop_indexes())
request.addfinalizer(sync_drop_indexes)
return client
async def make_es_index_and_run_sync(env: Env,
clients=32,
force=False,
db=None,
index_name='claims'):
index = SearchIndex(env.es_index_prefix,
elastic_host=env.elastic_host,
elastic_port=env.elastic_port)
logging.info("ES sync host: %s:%i", env.elastic_host, env.elastic_port)
try:
created = await index.start()
except IndexVersionMismatch as err:
logging.info(
"dropping ES search index (version %s) for upgrade to version %s",
err.got_version, err.expected_version)
await index.delete_index()
await index.stop()
created = await index.start()
finally:
index.stop()
es = AsyncElasticsearch([{
'host': env.elastic_host,
'port': env.elastic_port
}])
if force or created:
claim_generator = get_all_claims(env, index_name=index_name, db=db)
else:
claim_generator = get_recent_claims(env, index_name=index_name, db=db)
try:
async for ok, item in async_streaming_bulk(es,
claim_generator,
request_timeout=600,
raise_on_error=False):
if not ok:
logging.warning("indexing failed for an item: %s", item)
await es.indices.refresh(index=index_name)
finally:
await es.close()
def __init__(self,
group: PVGroup,
url: str,
skip_attributes: Optional[set] = None,
es: AsyncElasticsearch = None,
index: Optional[str] = None,
index_suffix: Optional[str] = None,
restore_on_startup: bool = True,
):
self.group = group
default_idx = inflection.underscore(
f'{group.name}-{group.prefix}'.replace(':', '_')
).lower()
self.index = index or default_idx
self.index_suffix = index_suffix or ''
self.index_search_glob = f'{self.index}*',
self.group.log.info(
'%s using elastic index %r (suffix %r)',
group, self.index, self.index_suffix
)
self._dated_index = None
self.get_last_document_query = {
'sort': {self.TIMESTAMP_KEY: 'desc'}
}
self.skip_attributes = skip_attributes or {}
if es is None:
es = AsyncElasticsearch([url])
self.es = es
self.restore_on_startup = restore_on_startup
self._restoring = False
logger.warning(
'Elasticsearch: %s Index: %r %s',
self.es, self.index,
f'(suffix {self.index_suffix!r})' if self.index_suffix else ''
)
async def clean(self, cleanups):
self.logger.debug(cleanups)
actions = CleanUpResult()
for index in itertools.chain.from_iterable(cleanups):
args = self.indexers[index["environment"]]["args"]
async with AsyncElasticsearch(**args) as client:
await client.indices.delete(index["_id"],
ignore_unavailable=True)
action = ("DELETE", str(index))
actions.append(action)
logging.info(action)
self.collection.update_many(
{
f"index.{index['_id']}.environment":
index["environment"]
}, {"$unset": {
f"index.{index['_id']}": 1
}})
return actions
async def get_connection() -> AsyncElasticsearch:
"""
Returns a connection to the Elasticsearch.
The connection is cached and reused. It is not
creating a new connection every time.
Required environment variables to create a connection:
- ENGINE_HOST
- ENGINE_PORT
- ENGINE_USE_SSL
"""
global _client
if _client:
return _client
client = AsyncElasticsearch(
hosts=[es_host],
port=int(es_port),
use_ssl=bool(strtobool(es_use_ssl)),
verify_certs=True,
http_compress=True,
max_retries=5,
retry_on_timeout=True,
)
_client = client
return client
import asyncio
from bitcoinrpc import BitcoinRPC
from pycoin.encoding.hexbytes import h2b
from pycoin.symbols.tbtx import network
from elasticsearch import AsyncElasticsearch
from dtabase import db, Transaction, create_transaction
from utils import serialize_transaction
Block = network.block
rpc = BitcoinRPC("127.0.0.1", 18332, "bc_test", "Emh67ztmLs2r5tdRUF9b")
es = AsyncElasticsearch()
async def main():
await db.set_bind('postgres://*****:*****@localhost:5432/blockchain')
await db.gino.create_all()
block_number = 0
while (block_number := block_number + 1) < await rpc.getblockcount():
block = await rpc.getblockhash(block_number)
block_hash = await rpc.getblock(block, verbosity=0)
block_data = Block.parse(io.BytesIO(h2b(block_hash)),
include_transactions=1)
for ix, tx in enumerate(block_data.txs):
if await Transaction.query.where(Transaction.tx_hash == tx.id()
