def post_to_es(payload):
'''Post data to ES cluster with exponential backoff'''
# Get aws_region and credentials to post signed URL to ES
es_region = ES_REGION or os.environ['AWS_REGION']
session = Session({'region': es_region})
creds = get_credentials(session)
es_url = urlparse.urlparse(ES_ENDPOINT)
es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT
# Post data with exponential backoff
retries = 0
while retries < ES_MAX_RETRIES:
if retries > 0:
seconds = (2 ** retries) * .1
time.sleep(seconds)
try:
es_ret_str = post_data_to_es(payload, es_region, creds, es_endpoint, '/_bulk')
es_ret = json.loads(es_ret_str)
if es_ret['errors']:
logger.error('ES post unsuccessful, errors present, took=%sms', es_ret['took'])
# Filter errors
es_errors = [item for item in es_ret['items'] if item.get('index').get('error')]
logger.error('List of items with errors: %s', json.dumps(es_errors))
else:
logger.info('ES post successful, took=%sms', es_ret['took'])
break # Sending to ES was ok, break retry loop
except ES_Exception as e:
if (e.status_code >= 500) and (e.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
def post_to_es(payload):
'''Post data to ES cluster with exponential backoff'''
# 署名されたURLをESに登録するため、aws_regionと認証情報を取得する
es_region = ES_REGION or os.environ['AWS_REGION']
session = Session({'region': es_region})
creds = get_credentials(session)
es_url = urlparse(ES_ENDPOINT)
es_endpoint = es_url.netloc or es_url.path # ESのエンドポイントでドメインを抽出
# エクスポネンシャルバックオフによるデータの登録
retries = 0
while retries < ES_MAX_RETRIES:
if retries > 0:
seconds = (2 ** retries) * .1
logger.debug('Waiting for %.1f seconds', seconds)
time.sleep(seconds)
try:
es_ret_str = post_data_to_es(payload, es_region, creds, es_endpoint, '/_bulk')
logger.debug('Return from ES: %s', es_ret_str)
es_ret = json.loads(es_ret_str)
if es_ret['errors']:
logger.error('ES post unsuccessful, errors present, took=%sms', es_ret['took'])
es_errors = [item for item in es_ret['items'] if item.get('index').get('error')]
logger.error('List of items with errors: %s', json.dumps(es_errors))
else:
logger.info('ES post successful, took=%sms', es_ret['took'])
break # ESへの登録が完了したら、リトライループを中断
except ES_Exception as e:
if (e.status_code >= 500) and (e.status_code <= 599):
retries += 1 # 再試行カウント
else:
raise # 再試行を停止し、例外を再発行
def lambda_handler(funcName, event, context):
try:
# Need to marshall the string into something we can get to in the
# Go universe, so for that we can just get a struct
# with the context. The event content can be passed in as a
# raw char pointer.
request = dict(event=event)
contextDict = dict(
functionName=context.function_name,
functionVersion=context.function_version,
invokedFunctionArn=context.invoked_function_arn,
memoryLimitInMB=context.memory_limit_in_mb,
awsRequestId=context.aws_request_id,
logGroupName=context.log_group_name,
logStreamName=context.log_stream_name
)
# Identity check...
if getattr(context, "identity", None):
contextDict["identity"] = dict(
cognitoIdentityId=context.identity.cognito_identity_id,
cognitoIdentityPoolId=context.identity.cognito_identity_pool_id
)
# Client context
if getattr(context, "client_context", None):
contextDict["client_context"] = dict(
installation_id=context.client_context.installation_id,
app_title=context.client_context.app_title,
app_version_name=context.client_context.app_version_name,
app_version_code=context.client_context.app_version_code,
Custom=context.client_context.custom,
env=context.client_context.env
)
# Update it
request["context"] = contextDict
memset(response_buffer, 0, MAX_RESPONSE_SIZE)
memset(response_content_type_buffer, 0, MAX_RESPONSE_CONTENT_TYPE_SIZE)
exitCode = c_int()
credentials = get_credentials(get_session())
bytesWritten = lib.Lambda(funcName.encode('utf-8'),
json.dumps(request).encode('utf-8'),
credentials.access_key.encode('utf-8'),
credentials.secret_key.encode('utf-8'),
credentials.token.encode('utf-8'),
byref(exitCode),
response_content_type_buffer,
MAX_RESPONSE_CONTENT_TYPE_SIZE-1,
response_buffer,
MAX_RESPONSE_SIZE-1)
lowercase_content_type = response_content_type_buffer.value.lower()
if "json" in lowercase_content_type.decode('utf-8'):
return json.loads(response_buffer.value)
else:
return response_buffer.value.decode('utf-8')
except:
traceback.print_exc()
print("Unexpected error:", sys.exc_info()[0])
def __auth(self):
creds = get_credentials(Session())
aws_auth = AWS4Auth(creds.access_key,
creds.secret_key,
self.region,
self.aws_type,
session_token=creds.token)
return aws_auth
def setup_aws_auth(region: str, profile: str = None) -> AWS4Auth:
"""For a given region sign a request to the execute-api with standard credentials."""
credentials = get_credentials(Session(profile=profile))
auth = AWS4Auth(credentials.access_key,
credentials.secret_key,
region,
"execute-api",
session_token=credentials.token)
return auth
def __init__(self, configuration):
super(AmazonElasticsearchService, self).__init__(configuration)
region = configuration['region']
cred = None
if configuration.get('use_aws_iam_profile', False):
cred = credentials.get_credentials(session.Session())
else:
cred = credentials.Credentials(access_key=configuration.get('access_key', ''),
secret_key=configuration.get('secret_key', ''))
self.auth = AWSV4Sign(cred, region, 'es')
def __init__(self, configuration):
super(AmazonElasticsearchService, self).__init__(configuration)
region = configuration['region']
cred = None
if configuration.get('use_aws_iam_profile', False):
cred = credentials.get_credentials(session.Session())
else:
cred = credentials.Credentials(
access_key=configuration.get('access_key', ''),
secret_key=configuration.get('secret_key', ''))
self.auth = AWSV4Sign(cred, region, 'es')
def __init__(self, configuration):
super(AmazonElasticsearchService, self).__init__(configuration)
region = configuration["region"]
cred = None
if configuration.get("use_aws_iam_profile", False):
cred = credentials.get_credentials(session.Session())
else:
cred = credentials.Credentials(
access_key=configuration.get("access_key", ""),
secret_key=configuration.get("secret_key", ""),
)
self.auth = AWSV4Sign(cred, region, "es")
def _postdeployment_handler(event, context):
logger.debug('Event: %s', event)
# Get aws_region and credentials to post signed URL to ES
session = Session({'region': ES_REGION})
creds = get_credentials(session)
es_url = urlparse(ES_ENDPOINT)
# Extract the domain name in ES_ENDPOINT
es_endpoint = es_url.netloc or es_url.path
# GET /<index> should return a 404 - if not, then return.
try:
logger.debug('GET /%s', ES_INDEX)
es_ret_str = post_data_to_es('', ES_REGION, creds, es_endpoint,
'/' + ES_INDEX, 'GET')
logger.debug('Return from ES: %s', es_ret_str)
logger.info('Index already exists - aborting')
return 'Completed - Index exists'
except ES_Exception as e:
if (e.status_code == 404):
logger.info(
'Index %s does not exist - continuing to configuration stage',
ES_INDEX)
else:
raise # re-raise exception
# PUT /<index> with payload to create the index
logger.debug('PUT /%s', ES_INDEX)
index_payload = json.dumps({
'settings': {
'index': {
'number_of_shards': 5,
'number_of_replicas': 1
}
},
'mappings': {
'doc': {
'properties': {
'gps': {
'type': 'geo_point'
}
}
}
}
})
es_ret_str = post_data_to_es(index_payload, ES_REGION, creds, es_endpoint,
'/' + ES_INDEX, 'PUT')
logger.debug('Return from ES: %s', es_ret_str)
return es_ret_str
def post_to_es(self, payload):
"""
High-level POST data to Amazon Elasticsearch Service with exponential backoff
according to suggested algorithm:
http://docs.aws.amazon.com/general/latest/gr/api-retries.html
:param payload:
:return:
"""
# Get aws_region and credentials to post signed URL to ES
es_region = self.es_region
session = Session({'region': es_region})
creds = get_credentials(session)
es_url = urlparse.urlparse(self.es_endpoint)
es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT
# Post data with exponential backoff
retries = 0
while retries < self.es_max_retries:
if retries > 0:
seconds = (2**retries) * .1
time.sleep(seconds)
try:
es_ret_str = self.post_data_to_es(payload, es_region, creds,
es_endpoint, '/_bulk')
es_ret = json.loads(es_ret_str)
if es_ret['errors']:
get_logger().error(
'ES post unsuccessful, errors present, took=%sms',
es_ret['took'])
# Filter errors
es_errors = [
item for item in es_ret['items']
if item.get('index').get('error')
]
get_logger().error('List of items with errors: %s',
json.dumps(es_errors))
else:
get_logger().info('ES post successful, took=%sms',
es_ret['took'])
break # Sending to ES was ok, break retry loop
except ESException as e:
if (e.status_code >= 500) and (e.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
def post_to_opensearch(payload):
'''Post data to OpenSearch cluster with exponential backoff'''
# Get aws_region and credentials to post signed URL to OpenSearch
opensearch_region = OPENSEARCH_REGION or os.environ['AWS_REGION']
session = Session({'region': opensearch_region})
creds = get_credentials(session)
opensearch_url = urlparse(OPENSEARCH_ENDPOINT)
# Extract the domain name in OPENSEARCH_ENDPOINT
opensearch_endpoint = opensearch_url.netloc or opensearch_url.path
# Post data with exponential backoff
retries = 0
while retries < OPENSEARCH_MAX_RETRIES:
if retries > 0:
seconds = (2**retries) * .1
logger.debug('Waiting for %.1f seconds', seconds)
time.sleep(seconds)
try:
opensearch_ret_str = post_data_to_opensearch(
payload, opensearch_region, creds, opensearch_endpoint,
'/_bulk')
logger.debug('Return from OpenSearch: %s', opensearch_ret_str)
opensearch_ret = json.loads(opensearch_ret_str)
if opensearch_ret['errors']:
logger.error(
'OpenSearch post unsuccessful, errors present, took=%sms',
opensearch_ret['took'])
# Filter errors
opensearch_errors = [
item for item in opensearch_ret['items']
if item.get('index', {}).get('error')
]
logger.error('List of items with errors: %s',
json.dumps(opensearch_errors))
else:
logger.info('OpenSearch post successful, took=%sms',
opensearch_ret['took'])
break # Sending to OpenSearch was ok, break retry loop
except Searchable_Exception as e:
if (e.status_code >= 500) and (e.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
def head_index_from_es(index_name, method='HEAD', proto='https://'):
es_url = urlparse.urlparse(ES_ENDPOINT)
es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT
'''Post data to ES endpoint with SigV4 signed http headers'''
req = AWSRequest(method=method,
url=proto + es_endpoint + '/' + urllib.quote(index_name),
headers={'Host': es_endpoint})
es_region = ES_REGION or os.environ['AWS_REGION']
session = Session()
SigV4Auth(get_credentials(session), 'es',
os.environ['AWS_REGION']).add_auth(req)
http_session = URLLib3Session()
res = http_session.send(req.prepare())
if res.status_code >= 200 and res.status_code <= 299:
logger.info('Index %s do exists, continue update setting', index_name)
return True
else:
logger.warning('Index %s does not exists, need to create.', index_name)
return False
def get_auth(host, region):
credential = credentials.get_credentials(Session())
if hasattr(credential, 'access_key'):
return AWSRequestsAuth(aws_access_key=credential.access_key,
aws_secret_access_key=credential.secret_key,
aws_token=credential.token,
aws_host=host.split(':')[0],
aws_region=region,
aws_service='es')
if os.environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"):
credential = ContainerMetadataFetcher().retrieve_uri()
return AWSRequestsAuth(aws_access_key=credential['access_key'],
aws_secret_access_key=credential['secret_key'],
aws_token=credential['token'],
aws_host=host.split(':')[0],
aws_region=region,
aws_service='es')
return None
def put_data_to_es(payload, path, method='PUT', proto='https://'):
es_url = urlparse.urlparse(ES_ENDPOINT)
es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT
'''Post data to ES endpoint with SigV4 signed http headers'''
req = AWSRequest(method=method,
url=proto + es_endpoint + '/' + urllib.quote(path),
data=payload,
headers={
'Host': es_endpoint,
'Content-Type': 'application/json'
})
es_region = ES_REGION or os.environ['AWS_REGION']
session = Session()
SigV4Auth(get_credentials(session), 'es',
os.environ['AWS_REGION']).add_auth(req)
http_session = URLLib3Session()
res = http_session.send(req.prepare())
if res.status_code >= 200 and res.status_code <= 299:
return res._content
else:
raise ES_Exception(res.status_code, res._content)
def login(region: str) -> AWS4Auth:
"""For a given region sign a request to the execute-api with standard credentials."""
credentials = get_credentials(Session())
auth = AWS4Auth(credentials.access_key, credentials.secret_key, region,
"execute-api")
return auth
def lambda_handler(funcName, event, context):
try:
# Need to marshall the string into something we can get to in the
# Go universe, so for that we can just get a struct
# with the context. The event content can be passed in as a
# raw char pointer.
# Base64 encode the event data because that's what
# proto expects for bytes data
eventJSON = json.dumps(event).encode('utf-8')
base64string = base64.b64encode(eventJSON).decode('utf-8')
request = dict(event=base64string)
contextDict = dict(
functionName=context.function_name,
functionVersion=context.function_version,
invokedFunctionArn=context.invoked_function_arn,
memoryLimitInMb=context.memory_limit_in_mb,
awsRequestId=context.aws_request_id,
logGroupName=context.log_group_name,
logStreamName=context.log_stream_name
)
# Identity check...
identityContext = getattr(context, "identity", None)
if identityContext is not None:
identityDict = dict()
if getattr(identityDict, "cognito_identity_id", None):
identityDict["cognitoIdentityId"] = identityContext["cognito_identity_id"]
if getattr(identityDict, "cognito_identity_pool_id", None):
identityDict["cognitoIdentityPoolId"] = identityContext["cognito_identity_pool_id"]
contextDict["identity"] = identityDict
# Client context
if getattr(context, "client_context", None):
awsClientContext = context.client_context
contextDict["client_context"] = dict(
installation_id=awsClientContext.installation_id,
app_title=awsClientContext.app_title,
app_version_name=awsClientContext.app_version_name,
app_version_code=awsClientContext.app_version_code,
Custom=awsClientContext.custom,
env=awsClientContext.env
)
# Update it
request["context"] = contextDict
memset(response_buffer, 0, MAX_RESPONSE_SIZE)
memset(response_content_type_buffer, 0, MAX_RESPONSE_CONTENT_TYPE_SIZE)
exitCode = c_int()
credentials = get_credentials(get_session())
logger.debug('Sending event: {}'.format(request))
bytesWritten = libHandle.Lambda(funcName.encode('utf-8'),
SPARTA_LOG_LEVEL.encode('utf-8'),
json.dumps(request).encode('utf-8'),
credentials.access_key.encode('utf-8'),
credentials.secret_key.encode('utf-8'),
credentials.token.encode('utf-8'),
byref(exitCode),
response_content_type_buffer,
MAX_RESPONSE_CONTENT_TYPE_SIZE-1,
response_buffer,
MAX_RESPONSE_SIZE-1)
logger.debug('Lambda exit code: {}'.format(exitCode))
if exitCode.value != 0:
raise Exception(response_content_type_buffer.value)
lowercase_content_type = response_content_type_buffer.value.lower()
if "json" in lowercase_content_type.decode('utf-8'):
try:
json_object = json.loads(response_buffer.value)
return json_object
except:
# They claim it's JSON, but it's not. Be nice
return response_buffer.value.decode('utf-8')
else:
return response_buffer.value.decode('utf-8')
except:
traceback.print_exc()
print("ERROR:", sys.exc_info()[0])
raise
import boto3
import requests
from botocore.credentials import get_credentials
from botocore.session import Session
from requests_aws4auth import AWS4Auth
host = 'EDITME with slash ->/'
region = 'us-east-1'
service = 'es'
bucket = 'EDITME'
role_arn = 'EDITME'
creds = get_credentials(Session())
awsauth = AWS4Auth(creds.access_key, creds.secret_key, region, service)
# essnapshot is like an alias, you can choose any name (ES >7.x API does not like - in the name)
path = '_snapshot/essnapshot'
url = host + path
payload = {
"type": "s3",
"settings": {
"bucket": bucket,
"region": region,
"role_arn": role_arn
}
}
headers = {"Content-Type": "application/json"}
