def post_to_es(payload): '''Post data to ES cluster with exponential backoff''' # Get aws_region and credentials to post signed URL to ES es_region = ES_REGION or os.environ['AWS_REGION'] session = Session({'region': es_region}) creds = get_credentials(session) es_url = urlparse.urlparse(ES_ENDPOINT) es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT # Post data with exponential backoff retries = 0 while retries < ES_MAX_RETRIES: if retries > 0: seconds = (2 ** retries) * .1 time.sleep(seconds) try: es_ret_str = post_data_to_es(payload, es_region, creds, es_endpoint, '/_bulk') es_ret = json.loads(es_ret_str) if es_ret['errors']: logger.error('ES post unsuccessful, errors present, took=%sms', es_ret['took']) # Filter errors es_errors = [item for item in es_ret['items'] if item.get('index').get('error')] logger.error('List of items with errors: %s', json.dumps(es_errors)) else: logger.info('ES post successful, took=%sms', es_ret['took']) break # Sending to ES was ok, break retry loop except ES_Exception as e: if (e.status_code >= 500) and (e.status_code <= 599): retries += 1 # Candidate for retry else: raise # Stop retrying, re-raise exception
def post_to_es(payload): '''Post data to ES cluster with exponential backoff''' # 署名されたURLをESに登録するため、aws_regionと認証情報を取得する es_region = ES_REGION or os.environ['AWS_REGION'] session = Session({'region': es_region}) creds = get_credentials(session) es_url = urlparse(ES_ENDPOINT) es_endpoint = es_url.netloc or es_url.path # ESのエンドポイントでドメインを抽出 # エクスポネンシャルバックオフによるデータの登録 retries = 0 while retries < ES_MAX_RETRIES: if retries > 0: seconds = (2 ** retries) * .1 logger.debug('Waiting for %.1f seconds', seconds) time.sleep(seconds) try: es_ret_str = post_data_to_es(payload, es_region, creds, es_endpoint, '/_bulk') logger.debug('Return from ES: %s', es_ret_str) es_ret = json.loads(es_ret_str) if es_ret['errors']: logger.error('ES post unsuccessful, errors present, took=%sms', es_ret['took']) es_errors = [item for item in es_ret['items'] if item.get('index').get('error')] logger.error('List of items with errors: %s', json.dumps(es_errors)) else: logger.info('ES post successful, took=%sms', es_ret['took']) break # ESへの登録が完了したら、リトライループを中断 except ES_Exception as e: if (e.status_code >= 500) and (e.status_code <= 599): retries += 1 # 再試行カウント else: raise # 再試行を停止し、例外を再発行
def lambda_handler(funcName, event, context): try: # Need to marshall the string into something we can get to in the # Go universe, so for that we can just get a struct # with the context. The event content can be passed in as a # raw char pointer. request = dict(event=event) contextDict = dict( functionName=context.function_name, functionVersion=context.function_version, invokedFunctionArn=context.invoked_function_arn, memoryLimitInMB=context.memory_limit_in_mb, awsRequestId=context.aws_request_id, logGroupName=context.log_group_name, logStreamName=context.log_stream_name ) # Identity check... if getattr(context, "identity", None): contextDict["identity"] = dict( cognitoIdentityId=context.identity.cognito_identity_id, cognitoIdentityPoolId=context.identity.cognito_identity_pool_id ) # Client context if getattr(context, "client_context", None): contextDict["client_context"] = dict( installation_id=context.client_context.installation_id, app_title=context.client_context.app_title, app_version_name=context.client_context.app_version_name, app_version_code=context.client_context.app_version_code, Custom=context.client_context.custom, env=context.client_context.env ) # Update it request["context"] = contextDict memset(response_buffer, 0, MAX_RESPONSE_SIZE) memset(response_content_type_buffer, 0, MAX_RESPONSE_CONTENT_TYPE_SIZE) exitCode = c_int() credentials = get_credentials(get_session()) bytesWritten = lib.Lambda(funcName.encode('utf-8'), json.dumps(request).encode('utf-8'), credentials.access_key.encode('utf-8'), credentials.secret_key.encode('utf-8'), credentials.token.encode('utf-8'), byref(exitCode), response_content_type_buffer, MAX_RESPONSE_CONTENT_TYPE_SIZE-1, response_buffer, MAX_RESPONSE_SIZE-1) lowercase_content_type = response_content_type_buffer.value.lower() if "json" in lowercase_content_type.decode('utf-8'): return json.loads(response_buffer.value) else: return response_buffer.value.decode('utf-8') except: traceback.print_exc() print("Unexpected error:", sys.exc_info()[0])
def __auth(self): creds = get_credentials(Session()) aws_auth = AWS4Auth(creds.access_key, creds.secret_key, self.region, self.aws_type, session_token=creds.token) return aws_auth
def setup_aws_auth(region: str, profile: str = None) -> AWS4Auth: """For a given region sign a request to the execute-api with standard credentials.""" credentials = get_credentials(Session(profile=profile)) auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, "execute-api", session_token=credentials.token) return auth
def __init__(self, configuration): super(AmazonElasticsearchService, self).__init__(configuration) region = configuration['region'] cred = None if configuration.get('use_aws_iam_profile', False): cred = credentials.get_credentials(session.Session()) else: cred = credentials.Credentials(access_key=configuration.get('access_key', ''), secret_key=configuration.get('secret_key', '')) self.auth = AWSV4Sign(cred, region, 'es')
def __init__(self, configuration): super(AmazonElasticsearchService, self).__init__(configuration) region = configuration['region'] cred = None if configuration.get('use_aws_iam_profile', False): cred = credentials.get_credentials(session.Session()) else: cred = credentials.Credentials( access_key=configuration.get('access_key', ''), secret_key=configuration.get('secret_key', '')) self.auth = AWSV4Sign(cred, region, 'es')
def __init__(self, configuration): super(AmazonElasticsearchService, self).__init__(configuration) region = configuration["region"] cred = None if configuration.get("use_aws_iam_profile", False): cred = credentials.get_credentials(session.Session()) else: cred = credentials.Credentials( access_key=configuration.get("access_key", ""), secret_key=configuration.get("secret_key", ""), ) self.auth = AWSV4Sign(cred, region, "es")
def _postdeployment_handler(event, context): logger.debug('Event: %s', event) # Get aws_region and credentials to post signed URL to ES session = Session({'region': ES_REGION}) creds = get_credentials(session) es_url = urlparse(ES_ENDPOINT) # Extract the domain name in ES_ENDPOINT es_endpoint = es_url.netloc or es_url.path # GET /<index> should return a 404 - if not, then return. try: logger.debug('GET /%s', ES_INDEX) es_ret_str = post_data_to_es('', ES_REGION, creds, es_endpoint, '/' + ES_INDEX, 'GET') logger.debug('Return from ES: %s', es_ret_str) logger.info('Index already exists - aborting') return 'Completed - Index exists' except ES_Exception as e: if (e.status_code == 404): logger.info( 'Index %s does not exist - continuing to configuration stage', ES_INDEX) else: raise # re-raise exception # PUT /<index> with payload to create the index logger.debug('PUT /%s', ES_INDEX) index_payload = json.dumps({ 'settings': { 'index': { 'number_of_shards': 5, 'number_of_replicas': 1 } }, 'mappings': { 'doc': { 'properties': { 'gps': { 'type': 'geo_point' } } } } }) es_ret_str = post_data_to_es(index_payload, ES_REGION, creds, es_endpoint, '/' + ES_INDEX, 'PUT') logger.debug('Return from ES: %s', es_ret_str) return es_ret_str
def post_to_es(self, payload): """ High-level POST data to Amazon Elasticsearch Service with exponential backoff according to suggested algorithm: http://docs.aws.amazon.com/general/latest/gr/api-retries.html :param payload: :return: """ # Get aws_region and credentials to post signed URL to ES es_region = self.es_region session = Session({'region': es_region}) creds = get_credentials(session) es_url = urlparse.urlparse(self.es_endpoint) es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT # Post data with exponential backoff retries = 0 while retries < self.es_max_retries: if retries > 0: seconds = (2**retries) * .1 time.sleep(seconds) try: es_ret_str = self.post_data_to_es(payload, es_region, creds, es_endpoint, '/_bulk') es_ret = json.loads(es_ret_str) if es_ret['errors']: get_logger().error( 'ES post unsuccessful, errors present, took=%sms', es_ret['took']) # Filter errors es_errors = [ item for item in es_ret['items'] if item.get('index').get('error') ] get_logger().error('List of items with errors: %s', json.dumps(es_errors)) else: get_logger().info('ES post successful, took=%sms', es_ret['took']) break # Sending to ES was ok, break retry loop except ESException as e: if (e.status_code >= 500) and (e.status_code <= 599): retries += 1 # Candidate for retry else: raise # Stop retrying, re-raise exception
def post_to_opensearch(payload): '''Post data to OpenSearch cluster with exponential backoff''' # Get aws_region and credentials to post signed URL to OpenSearch opensearch_region = OPENSEARCH_REGION or os.environ['AWS_REGION'] session = Session({'region': opensearch_region}) creds = get_credentials(session) opensearch_url = urlparse(OPENSEARCH_ENDPOINT) # Extract the domain name in OPENSEARCH_ENDPOINT opensearch_endpoint = opensearch_url.netloc or opensearch_url.path # Post data with exponential backoff retries = 0 while retries < OPENSEARCH_MAX_RETRIES: if retries > 0: seconds = (2**retries) * .1 logger.debug('Waiting for %.1f seconds', seconds) time.sleep(seconds) try: opensearch_ret_str = post_data_to_opensearch( payload, opensearch_region, creds, opensearch_endpoint, '/_bulk') logger.debug('Return from OpenSearch: %s', opensearch_ret_str) opensearch_ret = json.loads(opensearch_ret_str) if opensearch_ret['errors']: logger.error( 'OpenSearch post unsuccessful, errors present, took=%sms', opensearch_ret['took']) # Filter errors opensearch_errors = [ item for item in opensearch_ret['items'] if item.get('index', {}).get('error') ] logger.error('List of items with errors: %s', json.dumps(opensearch_errors)) else: logger.info('OpenSearch post successful, took=%sms', opensearch_ret['took']) break # Sending to OpenSearch was ok, break retry loop except Searchable_Exception as e: if (e.status_code >= 500) and (e.status_code <= 599): retries += 1 # Candidate for retry else: raise # Stop retrying, re-raise exception
def head_index_from_es(index_name, method='HEAD', proto='https://'): es_url = urlparse.urlparse(ES_ENDPOINT) es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT '''Post data to ES endpoint with SigV4 signed http headers''' req = AWSRequest(method=method, url=proto + es_endpoint + '/' + urllib.quote(index_name), headers={'Host': es_endpoint}) es_region = ES_REGION or os.environ['AWS_REGION'] session = Session() SigV4Auth(get_credentials(session), 'es', os.environ['AWS_REGION']).add_auth(req) http_session = URLLib3Session() res = http_session.send(req.prepare()) if res.status_code >= 200 and res.status_code <= 299: logger.info('Index %s do exists, continue update setting', index_name) return True else: logger.warning('Index %s does not exists, need to create.', index_name) return False
def get_auth(host, region): credential = credentials.get_credentials(Session()) if hasattr(credential, 'access_key'): return AWSRequestsAuth(aws_access_key=credential.access_key, aws_secret_access_key=credential.secret_key, aws_token=credential.token, aws_host=host.split(':')[0], aws_region=region, aws_service='es') if os.environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"): credential = ContainerMetadataFetcher().retrieve_uri() return AWSRequestsAuth(aws_access_key=credential['access_key'], aws_secret_access_key=credential['secret_key'], aws_token=credential['token'], aws_host=host.split(':')[0], aws_region=region, aws_service='es') return None
def put_data_to_es(payload, path, method='PUT', proto='https://'): es_url = urlparse.urlparse(ES_ENDPOINT) es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT '''Post data to ES endpoint with SigV4 signed http headers''' req = AWSRequest(method=method, url=proto + es_endpoint + '/' + urllib.quote(path), data=payload, headers={ 'Host': es_endpoint, 'Content-Type': 'application/json' }) es_region = ES_REGION or os.environ['AWS_REGION'] session = Session() SigV4Auth(get_credentials(session), 'es', os.environ['AWS_REGION']).add_auth(req) http_session = URLLib3Session() res = http_session.send(req.prepare()) if res.status_code >= 200 and res.status_code <= 299: return res._content else: raise ES_Exception(res.status_code, res._content)
def login(region: str) -> AWS4Auth: """For a given region sign a request to the execute-api with standard credentials.""" credentials = get_credentials(Session()) auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, "execute-api") return auth
def lambda_handler(funcName, event, context): try: # Need to marshall the string into something we can get to in the # Go universe, so for that we can just get a struct # with the context. The event content can be passed in as a # raw char pointer. # Base64 encode the event data because that's what # proto expects for bytes data eventJSON = json.dumps(event).encode('utf-8') base64string = base64.b64encode(eventJSON).decode('utf-8') request = dict(event=base64string) contextDict = dict( functionName=context.function_name, functionVersion=context.function_version, invokedFunctionArn=context.invoked_function_arn, memoryLimitInMb=context.memory_limit_in_mb, awsRequestId=context.aws_request_id, logGroupName=context.log_group_name, logStreamName=context.log_stream_name ) # Identity check... identityContext = getattr(context, "identity", None) if identityContext is not None: identityDict = dict() if getattr(identityDict, "cognito_identity_id", None): identityDict["cognitoIdentityId"] = identityContext["cognito_identity_id"] if getattr(identityDict, "cognito_identity_pool_id", None): identityDict["cognitoIdentityPoolId"] = identityContext["cognito_identity_pool_id"] contextDict["identity"] = identityDict # Client context if getattr(context, "client_context", None): awsClientContext = context.client_context contextDict["client_context"] = dict( installation_id=awsClientContext.installation_id, app_title=awsClientContext.app_title, app_version_name=awsClientContext.app_version_name, app_version_code=awsClientContext.app_version_code, Custom=awsClientContext.custom, env=awsClientContext.env ) # Update it request["context"] = contextDict memset(response_buffer, 0, MAX_RESPONSE_SIZE) memset(response_content_type_buffer, 0, MAX_RESPONSE_CONTENT_TYPE_SIZE) exitCode = c_int() credentials = get_credentials(get_session()) logger.debug('Sending event: {}'.format(request)) bytesWritten = libHandle.Lambda(funcName.encode('utf-8'), SPARTA_LOG_LEVEL.encode('utf-8'), json.dumps(request).encode('utf-8'), credentials.access_key.encode('utf-8'), credentials.secret_key.encode('utf-8'), credentials.token.encode('utf-8'), byref(exitCode), response_content_type_buffer, MAX_RESPONSE_CONTENT_TYPE_SIZE-1, response_buffer, MAX_RESPONSE_SIZE-1) logger.debug('Lambda exit code: {}'.format(exitCode)) if exitCode.value != 0: raise Exception(response_content_type_buffer.value) lowercase_content_type = response_content_type_buffer.value.lower() if "json" in lowercase_content_type.decode('utf-8'): try: json_object = json.loads(response_buffer.value) return json_object except: # They claim it's JSON, but it's not. Be nice return response_buffer.value.decode('utf-8') else: return response_buffer.value.decode('utf-8') except: traceback.print_exc() print("ERROR:", sys.exc_info()[0]) raise
import boto3 import requests from botocore.credentials import get_credentials from botocore.session import Session from requests_aws4auth import AWS4Auth host = 'EDITME with slash ->/' region = 'us-east-1' service = 'es' bucket = 'EDITME' role_arn = 'EDITME' creds = get_credentials(Session()) awsauth = AWS4Auth(creds.access_key, creds.secret_key, region, service) # essnapshot is like an alias, you can choose any name (ES >7.x API does not like - in the name) path = '_snapshot/essnapshot' url = host + path payload = { "type": "s3", "settings": { "bucket": bucket, "region": region, "role_arn": role_arn } } headers = {"Content-Type": "application/json"}