def _elasticsearch_connect(timeout=300):
"""
Connect to configured Elasticsearch domain.
:param timeout: How long to wait before ANY request to Elasticsearch times
out. Because we use parallel bulk uploads (which sometimes wait long periods
of time before beginning execution), a value of at least 30 seconds is
recommended.
:return: An Elasticsearch connection object.
"""
log.info('Connecting to %s %s with AWS auth', ELASTICSEARCH_URL,
ELASTICSEARCH_PORT)
auth = AWSRequestsAuth(aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_host=ELASTICSEARCH_URL,
aws_region=AWS_REGION,
aws_service='es')
auth.encode = lambda x: bytes(x.encode('utf-8'))
es = Elasticsearch(host=ELASTICSEARCH_URL,
port=ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=timeout,
max_retries=10,
retry_on_timeout=True,
http_auth=auth,
wait_for_status='yellow')
es.info()
return es
def _elasticsearch_connect():
"""
Connect to configured Elasticsearch domain.
:return: An Elasticsearch connection object.
"""
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
aws_host=settings.ELASTICSEARCH_URL,
aws_region=settings.ELASTICSEARCH_AWS_REGION,
aws_service='es'
)
auth.encode = lambda x: bytes(x.encode('utf-8'))
_es = Elasticsearch(
host=settings.ELASTICSEARCH_URL,
port=settings.ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=10,
max_retries=99,
retry_on_timeout=True,
http_auth=auth,
wait_for_status='yellow'
)
_es.info()
return _es
def _elasticsearch_connect() -> Elasticsearch:
"""
Connect to an Elasticsearch indices at the configured domain. This method also
handles AWS authentication using the AWS access key ID and the secret access key.
:return: an Elasticsearch client
"""
log.info(
f"Connecting to {ELASTICSEARCH_URL}:{ELASTICSEARCH_PORT} with AWS auth"
)
auth = AWSRequestsAuth(
aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_host=ELASTICSEARCH_URL,
aws_region=AWS_REGION,
aws_service="es",
)
auth.encode = lambda x: bytes(x.encode("utf-8"))
es = Elasticsearch(
host=ELASTICSEARCH_URL,
port=ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
http_auth=auth,
timeout=TWELVE_HOURS_SEC,
)
es.info()
return es
def _elasticsearch_connect():
"""
Connect to configured Elasticsearch domain.
:return: An Elasticsearch connection object.
"""
es_url = config("ELASTICSEARCH_URL", default="localhost")
es_port = config("ELASTICSEARCH_PORT", default=9200, cast=int)
es_aws_region = config("ELASTICSEARCH_AWS_REGION", default="us-east-1")
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
aws_host=es_url,
aws_region=es_aws_region,
aws_service="es",
)
auth.encode = lambda x: bytes(x.encode("utf-8"))
_es = Elasticsearch(
host=es_url,
port=es_port,
connection_class=RequestsHttpConnection,
timeout=10,
max_retries=1,
retry_on_timeout=True,
http_auth=auth,
wait_for_status="yellow",
)
_es.info()
return _es
def init_es(timeout=TIMEOUT):
log.info("connecting to %s %s", settings.ELASTICSEARCH_URL,
settings.ELASTICSEARCH_PORT)
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
aws_host=settings.ELASTICSEARCH_URL,
aws_region='us-west-1',
aws_service='es')
auth.encode = lambda x: bytes(x.encode('utf-8'))
es = Elasticsearch(host=settings.ELASTICSEARCH_URL,
port=settings.ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=timeout,
max_retries=10,
retry_on_timeout=True,
http_auth=auth)
return es
def _elasticsearch_connect():
"""
Connect to configured Elasticsearch domain.
:return: An Elasticsearch connection object.
"""
try:
log.info('Trying to connect to Elasticsearch without authentication...')
# Try to connect to Elasticsearch without credentials.
_es = Elasticsearch(
host=settings.ELASTICSEARCH_URL,
port=settings.ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=10,
max_retries=99,
wait_for_status='yellow'
)
log.info(str(_es.info()))
log.info('Connected to Elasticsearch without authentication.')
except (AuthenticationException, AuthorizationException):
# If that fails, supply AWS authentication object and try again.
log.info(
'Connecting to %s %s with AWS auth', settings.ELASTICSEARCH_URL,
settings.ELASTICSEARCH_PORT)
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
aws_host=settings.ELASTICSEARCH_URL,
aws_region=settings.ELASTICSEARCH_AWS_REGION,
aws_service='es'
)
auth.encode = lambda x: bytes(x.encode('utf-8'))
_es = Elasticsearch(
host=settings.ELASTICSEARCH_URL,
port=settings.ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=10,
max_retries=99,
retry_on_timeout=True,
http_auth=auth,
wait_for_status='yellow'
)
_es.info()
return _es
from aws_requests_auth.aws_auth import AWSRequestsAuth
import requests
import os
import datetime
import json
SNAPSHOT_DIR = 'ccsearch-snapshots'
# List all ES snapshots for this domain
if __name__ == "__main__":
auth = AWSRequestsAuth(
aws_access_key=os.environ['ES_AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['ES_AWS_SECRET_ACCESS_KEY'],
aws_host=os.environ['ES_CLUSTER_DNS'],
aws_region=os.environ['ES_REGION'],
aws_service='es')
auth.encode = lambda x: bytes(x.encode('utf-8'))
print("Listing available snapshots for {}".format(
os.environ['ES_CLUSTER_DNS']))
resp = requests.get('https://{}/_snapshot/{}/_all'.format(
os.environ['ES_CLUSTER_DNS'], SNAPSHOT_DIR),
auth=auth)
content = resp.json()
print(json.dumps(content, indent=4))
