def _elasticsearch_connect():
"""
Connect to configured Elasticsearch domain.
:return: An Elasticsearch connection object.
"""
es_url = config("ELASTICSEARCH_URL", default="localhost")
es_port = config("ELASTICSEARCH_PORT", default=9200, cast=int)
es_aws_region = config("ELASTICSEARCH_AWS_REGION", default="us-east-1")
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
aws_host=es_url,
aws_region=es_aws_region,
aws_service="es",
)
auth.encode = lambda x: bytes(x.encode("utf-8"))
_es = Elasticsearch(
host=es_url,
port=es_port,
connection_class=RequestsHttpConnection,
timeout=10,
max_retries=1,
retry_on_timeout=True,
http_auth=auth,
wait_for_status="yellow",
)
_es.info()
return _es
def __init__(self,
es_host,
region='us-east-1',
is_lambda=True,
index='iocs'):
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
if is_lambda:
aws_token = os.environ["AWS_SESSION_TOKEN"]
self.auth = AWSRequestsAuth(
aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_token=aws_token,
aws_host=es_host,
aws_region=region,
aws_service='es')
else:
self.auth = AWSRequestsAuth(
aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_host=es_host,
aws_region=region,
aws_service='es')
self.session = requests.session()
self.session.auth = self.auth
self.es_host = es_host
def __call__(self, r):
try:
# Host used in signature *MUST* always match with Host HTTP header.
host = r.headers.get('Host')
if not host:
_, _, host, _, _, _, _ = parse_url(r.url)
r.headers['Host'] = host
if self.domain is not None:
aws_params = self._parse_url(self.domain)
else:
aws_params = self._parse_url(host)
except ValueError:
aws_params = {"region": "us-west-2", "service": "svs"}
except Exception as error:
aws_params = {"region": "us-west-2", "service": "svs"}
aws_request = AWSRequestsAuth(
aws_access_key=self.aws_access_key,
aws_secret_access_key=self.aws_secret_access_key,
aws_host=host,
aws_region=aws_params['region'],
aws_service=aws_params['service'],
aws_token=self.aws_token)
return aws_request.__call__(r)
def _elasticsearch_connect() -> Elasticsearch:
"""
Connect to an Elasticsearch indices at the configured domain. This method also
handles AWS authentication using the AWS access key ID and the secret access key.
:return: an Elasticsearch client
"""
log.info(
f"Connecting to {ELASTICSEARCH_URL}:{ELASTICSEARCH_PORT} with AWS auth"
)
auth = AWSRequestsAuth(
aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_host=ELASTICSEARCH_URL,
aws_region=AWS_REGION,
aws_service="es",
)
auth.encode = lambda x: bytes(x.encode("utf-8"))
es = Elasticsearch(
host=ELASTICSEARCH_URL,
port=ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
http_auth=auth,
timeout=TWELVE_HOURS_SEC,
)
es.info()
return es
def _elasticsearch_connect(timeout=300):
"""
Connect to configured Elasticsearch domain.
:param timeout: How long to wait before ANY request to Elasticsearch times
out. Because we use parallel bulk uploads (which sometimes wait long periods
of time before beginning execution), a value of at least 30 seconds is
recommended.
:return: An Elasticsearch connection object.
"""
log.info('Connecting to %s %s with AWS auth', ELASTICSEARCH_URL,
ELASTICSEARCH_PORT)
auth = AWSRequestsAuth(aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_host=ELASTICSEARCH_URL,
aws_region=AWS_REGION,
aws_service='es')
auth.encode = lambda x: bytes(x.encode('utf-8'))
es = Elasticsearch(host=ELASTICSEARCH_URL,
port=ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=timeout,
max_retries=10,
retry_on_timeout=True,
http_auth=auth,
wait_for_status='yellow')
es.info()
return es
def __call__(self, r):
try:
# Host used in signature *MUST* always match with Host HTTP header.
host = r.headers.get('Host')
if not host:
_, _, host, _, _, _, _ = parse_url(r.url)
r.headers['Host'] = host
if self.domain is not None:
aws_params = self._parse_url(self.domain)
else:
aws_params = self._parse_url(host)
except ValueError:
print("ERROR: Could not parse neccessary information from URL.")
raise
except Exception as error:
print("Error parsing URL: %s" % error)
raise
if isinstance(r.body, io.RawIOBase):
r.body = r.body.readall()
elif isinstance(r.body, (io.BufferedIOBase, io.TextIOBase)):
r.body = r.body.read()
aws_request = AWSRequestsAuth(
aws_access_key=self.aws_access_key,
aws_secret_access_key=self.aws_secret_access_key,
aws_host=host,
aws_region=aws_params['region'],
aws_service=aws_params['service'],
aws_token=self.aws_token)
return aws_request.__call__(r)
def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:
if self.access_key is None:
fatal_plugin_error(f"ERROR: Missing {AWS_ACCESS_KEY} in environment")
if self.secret_key is None:
fatal_plugin_error(f"ERROR: Missing {AWS_SECRET_KEY} in environment")
try:
host = (
r.headers.get("Host").decode("utf-8")
if "Host" in r.headers
else urlparse(r.url).netloc
)
if self.region is None or self.service is None:
p = parse(
r"(?P<region>[^\.]+)\.(?P<service>.+)\.api\.nifcloud\.com", host
)
self.region = p["region"]
self.service = p["service"]
except Exception as error:
fatal_plugin_error(f"ERROR: {error}")
raise
auth = AWSRequestsAuth(
aws_access_key=self.access_key,
aws_secret_access_key=self.secret_key,
aws_host=host,
aws_region=self.region,
aws_service=self.service,
)
return auth.__call__(r)
def _elasticsearch_connect():
"""
Connect to configured Elasticsearch domain.
:return: An Elasticsearch connection object.
"""
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
aws_host=settings.ELASTICSEARCH_URL,
aws_region=settings.ELASTICSEARCH_AWS_REGION,
aws_service='es'
)
auth.encode = lambda x: bytes(x.encode('utf-8'))
_es = Elasticsearch(
host=settings.ELASTICSEARCH_URL,
port=settings.ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=10,
max_retries=99,
retry_on_timeout=True,
http_auth=auth,
wait_for_status='yellow'
)
_es.info()
return _es
def test_path_with_querystring(self):
"""
Assert we generate the 'correct' cannonical query string
and path for request that includes a query stirng
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/my_index/?pretty=True'
mock_request = mock.Mock()
mock_request.url = url
self.assertEqual('/my_index/', AWSRequestsAuth.get_caononical_path(mock_request))
self.assertEqual('pretty=True', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_characters_escaped_in_path(self):
"""
Assert we generate the 'correct' cannonical query string
and path a request with characters that need to be escaped
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/+foo.*/_stats'
mock_request = mock.Mock()
mock_request.url = url
self.assertEqual('/%2Bfoo.%2A/_stats', AWSRequestsAuth.get_caononical_path(mock_request))
self.assertEqual('', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_characters_escaped_in_path(self):
"""
Assert we generate the 'correct' cannonical query string
and path a request with characters that need to be escaped
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/+foo.*/_stats'
mock_request = mock.Mock()
mock_request.url = url
self.assertEqual('/%2Bfoo.%2A/_stats',
AWSRequestsAuth.get_canonical_path(mock_request))
self.assertEqual(
'', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_no_query_params(self):
"""
Assert we generate the 'correct' cannonical query string
and canonical path for a request with no query params
Correct is relative here b/c 'correct' simply means what
the AWS Elasticsearch service expects
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
self.assertEqual('/', AWSRequestsAuth.get_caononical_path(mock_request))
self.assertEqual('', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_path_with_querystring(self):
"""
Assert we generate the 'correct' cannonical query string
and path for request that includes a query stirng
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/my_index/?pretty=True'
mock_request = mock.Mock()
mock_request.url = url
self.assertEqual('/my_index/',
AWSRequestsAuth.get_canonical_path(mock_request))
self.assertEqual(
'pretty=True',
AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_no_query_params(self):
"""
Assert we generate the 'correct' cannonical query string
and canonical path for a request with no query params
Correct is relative here b/c 'correct' simply means what
the AWS Elasticsearch service expects
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
self.assertEqual('/', AWSRequestsAuth.get_canonical_path(mock_request))
self.assertEqual(
'', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_auth_for_post(self):
auth = AWSRequestsAuth(
aws_access_key='YOURKEY',
aws_secret_access_key='YOURSECRET',
aws_host='search-foo.us-east-1.es.amazonaws.com',
aws_region='us-east-1',
aws_service='es')
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
mock_request.method = "POST"
mock_request.body = b'foo=bar'
mock_request.headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
with mock.patch('datetime.datetime') as mock_datetime:
mock_datetime.utcnow.return_value = frozen_datetime
auth(mock_request)
self.assertEqual(
{
'Authorization':
'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=a6fd88e5f5c43e005482894001d9b05b43f6710e96be6098bcfcfccdeb8ed812',
'Content-Type':
'application/x-www-form-urlencoded',
'x-amz-date':
'20160618T220405Z',
}, mock_request.headers)
def test_auth_for_post_with_unicode_body_python3(self):
auth = AWSRequestsAuth(
aws_access_key='YOURKEY',
aws_secret_access_key='YOURSECRET',
aws_host='search-foo.us-east-1.es.amazonaws.com',
aws_region='us-east-1',
aws_service='es')
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
mock_request.method = "POST"
mock_request.body = 'foo=bar\xc3'
mock_request.headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
with mock.patch('datetime.datetime') as mock_datetime:
mock_datetime.utcnow.return_value = frozen_datetime
auth(mock_request)
self.assertEqual(
{
'Authorization':
'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=0836dae4bce95c1bcdbd3751c84c0c7e589ba7c81331bab92d0e1acb94adcdd9',
'Content-Type':
'application/x-www-form-urlencoded',
'x-amz-date':
'20160618T220405Z',
}, mock_request.headers)
def login(self, username, password):
"""
Retrieve the aws credentials used by the apis.
The login is staged, p1 we login to gigya, p2 we use p1 credentials
in irbt login api, that finally return aws credentials to
use in all the subsequent calls (mqtt and api gateway)
"""
# discover cloud settings
self._disc()
# login to gigya (get uid, timestamp and signature)
gigya_login = self._gigya_login(username, password)
# get aws keys through irbt login api
try:
(self.access_key_id,
self.secret_key,
self.session_token) = self._irbt_login_api(
gigya_login['UIDSignature'],
gigya_login['signatureTimestamp'],
gigya_login['UID'])
except KeyError:
logger.error('Authentication failed, wrong login/password')
return -1
# use AWSRequestsAuth module to set the aws authentication headers
# (Signature Version 4 Signing Process)
self.auth = AWSRequestsAuth(
aws_access_key=self.access_key_id,
aws_secret_access_key=self.secret_key,
aws_token=self.session_token,
aws_host=self._aws_host,
aws_region=self._aws_region,
aws_service='execute-api')
return True
def handler(event, context):
# For this function, we don't care about 'event' and 'context',
# but they need to be in the function signature anyway.
esinfo = {}
esendpoint = 'vpc-shop-v5-f4mgyu2gin24qdpouzn3icyz7m.eu-west-1.es.amazonaws.com'
session = boto3.session.Session()
credentials = session.get_credentials().get_frozen_credentials()
awsauth = AWSRequestsAuth(
aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=esendpoint,
aws_region=session.region_name,
aws_service='es'
)
es = Elasticsearch(
hosts=[{'host': esendpoint, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
ca_certs=certifi.where(),
connection_class=RequestsHttpConnection
)
esinfo = es.info();
print(esinfo)
def __init__(self):
self.session = requests.Session()
self.session.headers.update({
'X-Cuenca-Api-Version':
API_VERSION,
'User-Agent':
f'cuenca-python/{CLIENT_VERSION}',
})
# basic auth
api_key = os.getenv('CUENCA_API_KEY', '')
api_secret = os.getenv('CUENCA_API_SECRET', '')
self.basic_auth = (api_key, api_secret)
# IAM auth
aws_access_key = os.getenv('AWS_ACCESS_KEY_ID', '')
aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY', '')
aws_region = os.getenv('AWS_DEFAULT_REGION', AWS_DEFAULT_REGION)
if aws_access_key and aws_secret_access_key:
self.iam_auth = AWSRequestsAuth(
aws_access_key=aws_access_key,
aws_secret_access_key=aws_secret_access_key,
aws_host=self.host,
aws_region=aws_region,
aws_service=AWS_SERVICE,
)
def __init__(self, json_auth):
with open(json_auth) as f:
auth_info = json.load(f)
self.url = auth_info['apiEndpoint']
self.user_name = auth_info['userId']
host = self.url.split('/')[2]
self.navi_auth = AWSRequestsAuth(
aws_access_key=auth_info['accessKey'],
aws_secret_access_key=auth_info['secretKey'],
aws_host=host,
aws_region=auth_info['region'],
aws_service='execute-api')
#endpoints
self.status_endpoint = "api/v0/status"
self.robots_endpoint = "api/v0/users/{1}/robots"
self.robot_endpoint = "api/v0/users/{1}/robots/{2}"
self.sessions_endpoint = "api/v0/users/{1}/robots/{2}/sessions"
self.session_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}"
self.nodes_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes"
self.node_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}"
self.node_labelled_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/labelled/{4}"
self.odo_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/odometry"
self.big_data_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data"
self.big_data_element_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data/{5}"
self.big_data_raw_element_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data/{5}/raw"
self.variable_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/variables/{4}"
self.factors_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/factors"
self.factor_endpoint = "$factorsEndpoint/{4}"
self.bearing_range_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/factors/bearingrange"
self.session_ready_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/ready/{4}"
# context
self.sessionId = ""
self.robotId = ""
self.userId = ""
def _get_auth(self) -> AWSRequestsAuth:
"""Generate an AWSRequestsAuth object using a boto session for the current/local account.
Returns:
AWSRequestsAuth object
"""
if datetime.now() >= self._auth_expiration:
session = boto3.Session()
credentials = session.get_credentials()
region = (
session.region_name
if self._neptune_endpoint.region is None
else self._neptune_endpoint.region
)
auth = AWSRequestsAuth(
aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=f"{self._neptune_endpoint.host}:{self._neptune_endpoint.port}",
aws_region=region,
aws_service="neptune-db",
)
self._auth = auth
self._auth_expiration = datetime.now() + timedelta(minutes=SESSION_LIFETIME_MINUTES)
return self._auth
def connectES(esEndPoint):
print ('Connecting to the ES Endpoint {0}'.format(esEndPoint))
try:
access_key = os.getenv('AWS_ACCESS_KEY_ID')
secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
session_token = os.getenv('AWS_SESSION_TOKEN')
region = os.getenv('AWS_REGION')
auth = AWSRequestsAuth(aws_access_key=access_key,
aws_secret_access_key=secret_access_key,
aws_token=session_token,
aws_host=esEndPoint,
aws_region=region,
aws_service='es')
esClient = Elasticsearch(host=esEndPoint,
port=443,
use_ssl=True,
connection_class=RequestsHttpConnection,
http_auth=auth)
return esClient
except Exception as E:
print("Unable to connect to {0}".format(esEndPoint))
print(E)
exit(3)
def get_aws_auth():
"""Get HTTP authentication for our Elastic Search AWS user"""
return AWSRequestsAuth(aws_access_key=private.ES_ACCESS_KEY_ID,
aws_secret_access_key=private.ES_SECRET_ACCESS_KEY,
aws_host=private.ES_HOST,
aws_region=private.ES_REGION,
aws_service="es")
def amazonCall(url):
sts_client = boto3.client('sts')
# Call the assume_role method of the STSConnection object and pass the role
# ARN and a role session name.
assumedRoleObject = sts_client.assume_role(
RoleArn="arn:aws:iam::543303400543:role/MyApiCall",
RoleSessionName="mySession")
# From the response that contains the assumed role, get the temporary
# credentials that can be used to make subsequent API calls
credentials = assumedRoleObject['Credentials']
# Use the temporary credentials that AssumeRole returns to make a
# connection to Amazon S3
# s3_resource = boto3.resource(
# 's3',
# aws_access_key_id = ,
# aws_secret_access_key = ,
# aws_session_token = credentials['SessionToken'],
# )
# print("s3 resource:",s3_resource)
# # let's talk to our AWS Elasticsearch cluster
auth = AWSRequestsAuth(
aws_access_key=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_host='ec2-18-220-161-116.us-east-2.compute.amazonaws.com',
aws_region='us-east-2',
aws_service='compute')
response = requests.get(url, auth=auth)
return response
def lambda_handler(event, context):
print('Event Received: {}'.format(json.dumps(event)))
bucket_name = event['Records'][0]['s3']['bucket']['name']
# Wait for other objects to finish being uploaded
time.sleep(30)
resp = s3.list_objects_v2(Bucket=bucket_name)['Contents']
all_keys = [d['Key'] for d in resp]
final_keys = [
key for key in all_keys
if key.startswith('gisaid/') and key.endswith('vcf.gz')
]
locations = [f's3://{bucket_name}/{d}' for d in final_keys]
payload = {
"id": "gisaid",
"vcfLocations": locations,
}
api = BEACON_URL.split('/')[2]
url = BEACON_URL + "/submit"
auth = AWSRequestsAuth(
aws_access_key=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_token=aws_session_token,
aws_host=api,
aws_region='ap-southeast-2',
aws_service='execute-api',
)
response = requests.patch(url, data=json.dumps(payload), auth=auth)
print(dir(response))
def elasticsearch_client():
kwargs = {
'host': ES_HOST,
'port': ES_PORT,
}
if ES_SSL:
kwargs.update({
'use_ssl': True,
'ca_certs': certifi.where(),
})
if ES_SIGNING:
kwargs.update({
'connection_class':
RequestsHttpConnection,
'http_auth':
AWSRequestsAuth(aws_host=ES_HOST,
aws_region=ES_REGION,
aws_service='es',
**boto_utils.get_credentials()),
})
return Elasticsearch(**kwargs)
def test_auth_for_get(self):
auth = AWSRequestsAuth(
aws_access_key='YOURKEY',
aws_secret_access_key='YOURSECRET',
aws_host='search-foo.us-east-1.es.amazonaws.com',
aws_region='us-east-1',
aws_service='es')
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
mock_request.method = "GET"
mock_request.body = None
mock_request.headers = {}
frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
with mock.patch('datetime.datetime') as mock_datetime:
mock_datetime.utcnow.return_value = frozen_datetime
auth(mock_request)
self.assertEqual(
{
'Authorization':
'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=ca0a856286efce2a4bd96a978ca6c8966057e53184776c0685169d08abd74739',
'x-amz-date':
'20160618T220405Z',
}, mock_request.headers)
def _make_elastic(self):
"""create elasticsearch client"""
elastic_host = os.environ["ES_HOST"]
session = boto3.session.Session()
credentials = session.get_credentials().get_frozen_credentials()
awsauth = AWSRequestsAuth(
# These environment variables are automatically set by Lambda
aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=elastic_host,
aws_region=session.region_name,
aws_service="es"
)
return Elasticsearch(
hosts=[{"host": elastic_host, "port": 443}],
http_auth=awsauth,
max_backoff=get_time_remaining(self.context) if self.context else MAX_BACKOFF,
# Give ES time to respond when under load
timeout=ELASTIC_TIMEOUT,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
def get_elasticsearch_connection():
"gets an aws-authenticated elasticsearch connection"
session = boto3.session.Session()
credentials = session.get_credentials().get_frozen_credentials()
es_host = (
"search-sciwiki-search-0-f7ntx2mpc5g6yohp6dtiwzsdiy.us-west-2.es.amazonaws.com"
)
awsauth = AWSRequestsAuth(
aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=es_host,
aws_region=session.region_name,
aws_service="es",
)
return Elasticsearch(
hosts=[{
"host": es_host,
"port": 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
)
async def async_process_db_change(event, context):
aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
aws_session_token = os.environ['AWS_SESSION_TOKEN']
auth = AWSRequestsAuth(
aws_access_key=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_token=aws_session_token,
aws_host='0ppn9ycig5.execute-api.us-east-1.amazonaws.com',
aws_region='us-east-1',
aws_service='execute-api')
latest_record = {}
for i, record in enumerate(event['Records']):
latest_record[record['dynamodb']['Keys']['id']['S']] = i
all_tasks = []
for game_id, ind in latest_record.items():
if 'NewImage' not in event['Records'][ind]['dynamodb']:
continue
if not event['Records'][ind]['dynamodb']['NewImage']['id'][
'S'].startswith('game'):
continue
record = convert_record(event['Records'][ind]['dynamodb']['NewImage'])
try:
all_tasks.append(asyncio.create_task(handle_record(record, auth)))
except ValueError as e:
return {
'statusCode': 409,
'body': json.dumps('Game state not recognized')
}
await asyncio.gather(*all_tasks)
return {'statusCode': 200, 'body': json.dumps('success')}
def test_auth_for_post_with_unicode_body_python2(self):
auth = AWSRequestsAuth(
aws_access_key='YOURKEY',
aws_secret_access_key='YOURSECRET',
aws_host='search-foo.us-east-1.es.amazonaws.com',
aws_region='us-east-1',
aws_service='es')
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
mock_request.method = "POST"
mock_request.body = 'foo=bar\xc3'
mock_request.headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
with mock.patch('datetime.datetime') as mock_datetime:
mock_datetime.utcnow.return_value = frozen_datetime
auth(mock_request)
self.assertEqual(
{
'Authorization':
'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=88046be72423b267de5e7e604aaffb2c5668c3fd9022ef4aac8287b82ab71124',
'Content-Type':
'application/x-www-form-urlencoded',
'x-amz-date':
'20160618T220405Z',
}, mock_request.headers)
def lambda_handler(event, context):
es_host = os.environ['ELASTIC_SEARCH_HOSTNAME']
num_indexes_to_keep = int(os.environ['NUM_INDEXES_TO_KEEP'])
index_prefix = os.environ['INDEX_PREFIX']
auth = AWSRequestsAuth(aws_access_key=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
aws_token=os.environ['AWS_SESSION_TOKEN'],
aws_region=os.environ['AWS_REGION'],
aws_service='es',
aws_host=es_host)
indexResponse = requests.get(f"https://{es_host}/*", auth=auth)
if (indexResponse.status_code != 200):
raise Exception(f"failed to retrieve indexes: {indexResponse.text}")
indexData = indexResponse.json()
indexNames = sorted([x for x in indexData.keys() if x.startswith(index_prefix)])
indexesToDelete = indexNames[0 : max(0, len(indexNames) - num_indexes_to_keep)]
print(f"number of selected indexes: {len(indexNames)}, number to delete: {len(indexesToDelete)}")
for idx in indexesToDelete:
deleteResponse = requests.delete(f"https://{es_host}/{idx}", auth=auth)
if deleteResponse.status_code == 200:
print(f"deleted {idx}")
else:
raise Exception(f"failed to delete {idx}: {deleteResponse.text}")
def __create_connection(self, config):
kwargs = {
'host': config['HOST'],
'port': config.get('PORT', 9200),
'use_ssl': config.get('USE_SSL', False),
'verify_certs': True,
'ca_certs': certifi.where()
}
if 'AWS_ACCESS_KEY' in config and \
'AWS_SECRET_KEY' in config and \
'AWS_REGION' in config:
kwargs['connection_class'] = RequestsHttpConnection
kwargs['http_auth'] = AWSRequestsAuth(
aws_access_key=config['AWS_ACCESS_KEY'],
aws_secret_access_key=config['AWS_SECRET_KEY'],
aws_host=config['HOST'],
aws_region=config['AWS_REGION'],
aws_service='es')
es = Elasticsearch(**kwargs)
es._index = config['INDEX_NAME']
es._settings = config.get('INDEX_SETTINGS', DEFAULT_INDEX_SETTINGS)
return es
def test_multiple_get_params(self):
"""
Assert we generate the 'correct' cannonical query string
for request that includes more than one query parameter
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/index/type/_search?scroll=5m&search_type=scan'
mock_request = mock.Mock()
mock_request.url = url
self.assertEqual('scroll=5m&search_type=scan', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_post_request_with_get_param(self):
"""
Assert we generate the 'correct' cannonical query string
for a post request that includes GET-parameters
"""
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/index/type/1/_update?version=1'
mock_request = mock.Mock()
mock_request.url = url
mock_request.method = "POST"
self.assertEqual('version=1', AWSRequestsAuth.get_canonical_querystring(mock_request))
