def __call__(self, r):
try:
# Host used in signature *MUST* always match with Host HTTP header.
host = r.headers.get('Host')
if not host:
_, _, host, _, _, _, _ = parse_url(r.url)
r.headers['Host'] = host
if self.domain != None:
aws_params = self._parse_url(self.domain)
else:
aws_params = self._parse_url(host)
except ValueError:
print("ERROR: Could not parse neccessary information from URL.")
raise
except Exception as error:
print("Error parsing URL: %s" % error)
raise
aws_request = AWSRequestsAuth(
aws_access_key=self.aws_access_key,
aws_secret_access_key=self.aws_secret_access_key,
aws_host=host,
aws_region=aws_params['region'],
aws_service=aws_params['service'],
aws_token=self.aws_token)
return aws_request.__call__(r)
def connectES(esEndPoint):
print ('Connecting to the ES Endpoint {0}'.format(esEndPoint))
try:
access_key = os.getenv('AWS_ACCESS_KEY_ID')
secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
session_token = os.getenv('AWS_SESSION_TOKEN')
region = os.getenv('AWS_REGION')
auth = AWSRequestsAuth(aws_access_key=access_key,
aws_secret_access_key=secret_access_key,
aws_token=session_token,
aws_host=esEndPoint,
aws_region=region,
aws_service='es')
esClient = Elasticsearch(host=esEndPoint,
port=443,
use_ssl=True,
connection_class=RequestsHttpConnection,
http_auth=auth)
return esClient
except Exception as E:
print("Unable to connect to {0}".format(esEndPoint))
print(E)
exit(3)
def _lazy_haystack_setup():
from django.conf import settings
es_url = '%s%s' % (settings.ES_PROTOCOL, settings.ES_HOST)
if settings.AWS_ES_SIGN_REQUESTS:
from aws_requests_auth.aws_auth import AWSRequestsAuth
from elasticsearch import RequestsHttpConnection
auth = AWSRequestsAuth(
aws_host=settings.ES_HOST,
aws_region=config('AWS_ES_REGION', default=''),
aws_service='es',
**boto_utils.get_credentials()
)
haystack_connections = {
'default': {
'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
'URL': es_url,
'INDEX_NAME': config('ES_INDEX_NAME', default='mozillians_haystack'),
'KWARGS': {
'http_auth': auth,
'connection_class': RequestsHttpConnection,
}
}
}
else:
haystack_connections = {
'default': {
'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
'URL': es_url,
'INDEX_NAME': config('ES_INDEX_NAME', default='mozillians_haystack')
}
}
return haystack_connections
def _get_auth(self) -> AWSRequestsAuth:
"""Generate an AWSRequestsAuth object using a boto session for the current/local account.
Returns:
AWSRequestsAuth object
"""
if datetime.now() >= self._auth_expiration:
session = boto3.Session()
credentials = session.get_credentials()
region = (
session.region_name
if self._neptune_endpoint.region is None
else self._neptune_endpoint.region
)
auth = AWSRequestsAuth(
aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=f"{self._neptune_endpoint.host}:{self._neptune_endpoint.port}",
aws_region=region,
aws_service="neptune-db",
)
self._auth = auth
self._auth_expiration = datetime.now() + timedelta(minutes=SESSION_LIFETIME_MINUTES)
return self._auth
def lambda_handler(event, context):
print('Event Received: {}'.format(json.dumps(event)))
bucket_name = event['Records'][0]['s3']['bucket']['name']
# Wait for other objects to finish being uploaded
time.sleep(30)
resp = s3.list_objects_v2(Bucket=bucket_name)['Contents']
all_keys = [d['Key'] for d in resp]
final_keys = [
key for key in all_keys
if key.startswith('gisaid/') and key.endswith('vcf.gz')
]
locations = [f's3://{bucket_name}/{d}' for d in final_keys]
payload = {
"id": "gisaid",
"vcfLocations": locations,
}
api = BEACON_URL.split('/')[2]
url = BEACON_URL + "/submit"
auth = AWSRequestsAuth(
aws_access_key=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_token=aws_session_token,
aws_host=api,
aws_region='ap-southeast-2',
aws_service='execute-api',
)
response = requests.patch(url, data=json.dumps(payload), auth=auth)
print(dir(response))
def test_auth_for_post_with_str_body(self):
auth = AWSRequestsAuth(
aws_access_key='YOURKEY',
aws_secret_access_key='YOURSECRET',
aws_host='search-foo.us-east-1.es.amazonaws.com',
aws_region='us-east-1',
aws_service='es')
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
mock_request.method = "POST"
mock_request.body = 'foo=bar'
mock_request.headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
with mock.patch('datetime.datetime') as mock_datetime:
mock_datetime.utcnow.return_value = frozen_datetime
auth(mock_request)
self.assertEqual(
{
'Authorization':
'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=a6fd88e5f5c43e005482894001d9b05b43f6710e96be6098bcfcfccdeb8ed812',
'Content-Type':
'application/x-www-form-urlencoded',
'x-amz-date':
'20160618T220405Z',
}, mock_request.headers)
def _elasticsearch_connect(timeout=300):
"""
Connect to configured Elasticsearch domain.
:param timeout: How long to wait before ANY request to Elasticsearch times
out. Because we use parallel bulk uploads (which sometimes wait long periods
of time before beginning execution), a value of at least 30 seconds is
recommended.
:return: An Elasticsearch connection object.
"""
log.info('Connecting to %s %s with AWS auth', ELASTICSEARCH_URL,
ELASTICSEARCH_PORT)
auth = AWSRequestsAuth(aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_host=ELASTICSEARCH_URL,
aws_region=AWS_REGION,
aws_service='es')
auth.encode = lambda x: bytes(x.encode('utf-8'))
es = Elasticsearch(host=ELASTICSEARCH_URL,
port=ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=timeout,
max_retries=10,
retry_on_timeout=True,
http_auth=auth,
wait_for_status='yellow')
es.info()
return es
def test_auth_for_get(self):
auth = AWSRequestsAuth(
aws_access_key='YOURKEY',
aws_secret_access_key='YOURSECRET',
aws_host='search-foo.us-east-1.es.amazonaws.com',
aws_region='us-east-1',
aws_service='es')
url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
mock_request = mock.Mock()
mock_request.url = url
mock_request.method = "GET"
mock_request.body = None
mock_request.headers = {}
frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
with mock.patch('datetime.datetime') as mock_datetime:
mock_datetime.utcnow.return_value = frozen_datetime
auth(mock_request)
self.assertEqual(
{
'Authorization':
'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=ca0a856286efce2a4bd96a978ca6c8966057e53184776c0685169d08abd74739',
'x-amz-date':
'20160618T220405Z',
}, mock_request.headers)
def amazonCall(url):
sts_client = boto3.client('sts')
# Call the assume_role method of the STSConnection object and pass the role
# ARN and a role session name.
assumedRoleObject = sts_client.assume_role(
RoleArn="arn:aws:iam::543303400543:role/MyApiCall",
RoleSessionName="mySession")
# From the response that contains the assumed role, get the temporary
# credentials that can be used to make subsequent API calls
credentials = assumedRoleObject['Credentials']
# Use the temporary credentials that AssumeRole returns to make a
# connection to Amazon S3
# s3_resource = boto3.resource(
# 's3',
# aws_access_key_id = ,
# aws_secret_access_key = ,
# aws_session_token = credentials['SessionToken'],
# )
# print("s3 resource:",s3_resource)
# # let's talk to our AWS Elasticsearch cluster
auth = AWSRequestsAuth(
aws_access_key=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_host='ec2-18-220-161-116.us-east-2.compute.amazonaws.com',
aws_region='us-east-2',
aws_service='compute')
response = requests.get(url, auth=auth)
return response
def lambda_handler(event, context):
es_host = os.environ['ELASTIC_SEARCH_HOSTNAME']
num_indexes_to_keep = int(os.environ['NUM_INDEXES_TO_KEEP'])
index_prefix = os.environ['INDEX_PREFIX']
auth = AWSRequestsAuth(aws_access_key=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
aws_token=os.environ['AWS_SESSION_TOKEN'],
aws_region=os.environ['AWS_REGION'],
aws_service='es',
aws_host=es_host)
indexResponse = requests.get('https://' + es_host + '/*', auth=auth)
if (indexResponse.status_code != 200):
raise Exception('failed to retrieve indexes: ' + indexResponse.text)
indexData = indexResponse.json()
indexNames = sorted([x for x in indexData.keys() if x.startswith(index_prefix)])
indexesToDelete = indexNames[0 : max(0, len(indexNames) - num_indexes_to_keep)]
for idx in indexesToDelete:
deleteResponse = requests.delete('https://' + es_host + '/' + idx, auth=auth)
if deleteResponse.status_code == 200:
print("deleted " + idx)
else:
raise Exception("failed to delete " + idx + ": " + deleteResponse.text)
def _elasticsearch_connect():
"""
Connect to configured Elasticsearch domain.
:return: An Elasticsearch connection object.
"""
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
aws_host=settings.ELASTICSEARCH_URL,
aws_region=settings.ELASTICSEARCH_AWS_REGION,
aws_service='es'
)
auth.encode = lambda x: bytes(x.encode('utf-8'))
_es = Elasticsearch(
host=settings.ELASTICSEARCH_URL,
port=settings.ELASTICSEARCH_PORT,
connection_class=RequestsHttpConnection,
timeout=10,
max_retries=99,
retry_on_timeout=True,
http_auth=auth,
wait_for_status='yellow'
)
_es.info()
return _es
def __create_connection(self, config):
kwargs = {
'host': config['HOST'],
'port': config.get('PORT', 9200),
'use_ssl': config.get('USE_SSL', False),
'verify_certs': True,
'ca_certs': certifi.where()
}
if 'AWS_ACCESS_KEY' in config and \
'AWS_SECRET_KEY' in config and \
'AWS_REGION' in config:
kwargs['connection_class'] = RequestsHttpConnection
kwargs['http_auth'] = AWSRequestsAuth(
aws_access_key=config['AWS_ACCESS_KEY'],
aws_secret_access_key=config['AWS_SECRET_KEY'],
aws_host=config['HOST'],
aws_region=config['AWS_REGION'],
aws_service='es')
es = Elasticsearch(**kwargs)
es._index = config['INDEX_NAME']
es._settings = config.get('INDEX_SETTINGS', DEFAULT_INDEX_SETTINGS)
return es
def configure_connection():
"""Configure Elasticsearch default connection."""
if settings.ES_USE_AWS_AUTH:
es_protocol = {
'http': 80,
'https': 443,
}
es_host = urlparse(settings.ES_URL)
es_port = es_host.port if es_host.port else es_protocol.get(
es_host.scheme)
auth = AWSRequestsAuth(
aws_access_key=settings.AWS_ELASTICSEARCH_KEY,
aws_secret_access_key=settings.AWS_ELASTICSEARCH_SECRET,
aws_host=es_host.netloc,
aws_region=settings.AWS_ELASTICSEARCH_REGION,
aws_service='es',
)
connections_default = {
'hosts': [es_host.netloc],
'port': es_port,
'use_ssl': settings.ES_USE_SSL,
'verify_certs': settings.ES_VERIFY_CERTS,
'http_auth': auth,
'connection_class': RequestsHttpConnection,
}
else:
connections_default = {
'hosts': [settings.ES_URL],
'verify_certs': settings.ES_VERIFY_CERTS,
}
connections.configure(default=connections_default, )
def _make_elastic(self):
"""create elasticsearch client"""
elastic_host = os.environ["ES_HOST"]
session = boto3.session.Session()
credentials = session.get_credentials().get_frozen_credentials()
awsauth = AWSRequestsAuth(
# These environment variables are automatically set by Lambda
aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=elastic_host,
aws_region=session.region_name,
aws_service="es")
return Elasticsearch(
hosts=[{
"host": elastic_host,
"port": 443
}],
http_auth=awsauth,
max_backoff=get_time_remaining(self.context)
if self.context else MAX_BACKOFF,
# Give ES time to respond when under load
timeout=ELASTIC_TIMEOUT,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
def handler(event: Any, _context: Any) -> None:
# As per https://github.com/DavidMuller/aws-requests-auth#elasticsearch-py-client-usage-example
auth = AWSRequestsAuth(
aws_access_key=os.environ["AWS_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
aws_token=os.environ["AWS_SESSION_TOKEN"],
aws_host=es_host,
aws_region=region,
aws_service="es",
)
es_client = Elasticsearch(
host=es_host,
port=443,
use_ssl=True,
connection_class=RequestsHttpConnection,
http_auth=auth,
)
for record in event["Records"]:
bucket = record["s3"]["bucket"]["name"]
key = record["s3"]["object"]["key"]
if check_filename_fn(key):
common.s3_to_es(bucket=bucket,
key=key,
transform_fn=transform_fn,
es_client=es_client)
else:
logger.warning("Skipping object %r", key)
def handler(event, context):
global args
args = get_parameters()
global logger
logger = setup_logging(args.log_level)
logger.info(f"Event: {event}")
session = boto3.session.Session()
default_credentials = session.get_credentials().get_frozen_credentials()
lambda_client = boto3.client("lambda",
region_name=args.mismatch_lambda_region)
request_auth = AWSRequestsAuth(
aws_access_key=default_credentials.access_key,
aws_secret_access_key=default_credentials.secret_key,
aws_token=default_credentials.token,
aws_host=f"{args.api_hostname}",
aws_region=f"{args.api_region}",
aws_service="execute-api",
)
try:
original_request = json.loads(event["originalRequest"])
original_response = json.loads(event["originalResponse"])
except KeyError as e:
logger.error("Attempted to extract event items but was unable.")
logger.error(e)
raise e
actual_response = replay_original_request(request_auth, original_request,
args)
decrypted_original_response = decrypt_response(original_response,
original_request,
args.v2_kms_region)
decrypted_actual_response = decrypt_response(actual_response,
original_request,
args.v1_kms_region)
if compare_responses(
decrypted_original_response,
decrypted_actual_response,
original_request,
lambda_client,
):
logger.info(
f'Final result", "status": "match", '
f'"nino": "{original_request.get("nino")}", '
f'"transaction_id": "{original_request.get("transaction_id")}", '
f'"from_date": "{original_request.get("from_date")}", '
f'"to_date": "{original_request.get("to_date")}')
else:
logger.info(
f'Final result", "status": "miss", '
f'"nino": "{original_request.get("nino")}", '
f'"transaction_id": "{original_request.get("transaction_id")}", '
f'"from_date": "{original_request.get("from_date")}", '
f'"to_date": "{original_request.get("to_date")}')
def __init__(self,
thread_timeout=None,
chunk=None,
size=None,
is_lambda=True,
verbose_logging=False):
'''
an init, because reasons
'''
self.verbose_logging = verbose_logging
if thread_timeout is None:
thread_timeout = 10
self.process_timeout = thread_timeout
region = 'us-east-1' # e.g. us-east-1
if chunk is None:
chunk = 25
self.chunk = chunk
if size is None:
size = 600
self.size = size
self.procs = []
#self.process_timeout = 6
self.es_host = os.environ['ES_HOST']
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
if is_lambda:
aws_token = os.environ["AWS_SESSION_TOKEN"]
self.auth = AWSRequestsAuth(
aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_token=aws_token,
aws_host=self.es_host,
aws_region=region,
aws_service='es')
else:
self.auth = AWSRequestsAuth(
aws_access_key=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_host=self.es_host,
aws_region=region,
aws_service='es')
self.session = requests.session()
self.session.auth = self.auth
#ES 6.0+ requires explicit Content-Type headers
self.session.headers.update({"Content-Type": "application/json"})
self.whois_answers = []
def setUp(self):
self.trash_schedule_service = TrashScheduleService(
'https://test.amazonaws.com/Prod/v1/holidays')
self.auth = AWSRequestsAuth(aws_access_key='ACCESS_KEY',
aws_secret_access_key='SECRET',
aws_host='test.amazonaws.com',
aws_region='us-east1',
aws_service='execute-api')
def sign():
auth = AWSRequestsAuth(
aws_access_key=config.AWS_ACCESS_KEY_ID,
aws_secret_access_key=config.AWS_SECRET_ACCESS_KEY,
aws_host=config.HOST,
aws_region="us-east-1",
aws_service="execute-api",
)
return auth
def __set_aws_requests_auth(cls):
if cls.auth is None:
cls.auth = AWSRequestsAuth(
aws_access_key=os.environ['PRIVATE_CHAIN_AWS_ACCESS_KEY'],
aws_secret_access_key=os.
environ['PRIVATE_CHAIN_AWS_SECRET_ACCESS_KEY'],
aws_host=os.environ['PRIVATE_CHAIN_EXECUTE_API_HOST'],
aws_region='ap-northeast-1',
aws_service='execute-api')
def ask_lex(utterance, user):
url = 'https://runtime.lex.us-east-1.amazonaws.com/bot/First/alias/firstone/user/' + user + '/text'
auth = AWSRequestsAuth(aws_access_key=aws_access_key,
aws_secret_access_key=aws_secret_access_key,
aws_host='runtime.lex.us-east-1.amazonaws.com',
aws_region='us-east-1',
aws_service='lex')
data = {"inputText": utterance, "sessionAttributes": {"string": "string"}}
return requests.post(url=url, json=data, auth=auth)
def __create_new_account():
auth = AWSRequestsAuth(aws_access_key=os.environ['PRIVATE_CHAIN_AWS_ACCESS_KEY'],
aws_secret_access_key=os.environ['PRIVATE_CHAIN_AWS_SECRET_ACCESS_KEY'],
aws_host=os.environ['PRIVATE_CHAIN_EXECUTE_API_HOST'],
aws_region='ap-northeast-1',
aws_service='execute-api')
response = requests.post('https://' + os.environ['PRIVATE_CHAIN_EXECUTE_API_HOST'] +
'/production/accounts/new', auth=auth)
return json.loads(response.text)['result']
def _get_auth(self) -> AWSRequestsAuth:
host = urlparse(self.url).netloc
credentials = TrashScheduleService.__get_credentials()
return AWSRequestsAuth(aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=host,
aws_region=boto3.Session().region_name,
aws_service='execute-api')
def init_elasticsearch():
auth = AWSRequestsAuth(
aws_host=current_app.config["AWS_ELASTICSEARCH_HOST"],
aws_region=current_app.config["AWS_ELASTICSEARCH_REGION"],
aws_service="es",
**boto_utils.get_credentials())
return Elasticsearch(host=current_app.config["AWS_ELASTICSEARCH_HOST"],
port=current_app.config["AWS_ELASTICSEARCH_PORT"],
connection_class=RequestsHttpConnection,
http_auth=auth)
def get_aws_auth(url):
api_gateway_netloc = urlparse.urlparse(url).netloc
api_gateway_region = re.match(
r"[a-z0-9]+\.execute-api\.(.+)\.amazonaws\.com",
api_gateway_netloc).group(1)
return AWSRequestsAuth(aws_host=api_gateway_netloc,
aws_region=api_gateway_region,
aws_service='execute-api',
**boto_utils.get_credentials())
def get_aws_auth(self):
'''Use pip package to get IAM creds.'''
session = botocore.session.get_session()
aws_credentials = session.get_credentials()
auth = AWSRequestsAuth(aws_access_key=aws_credentials.access_key,
aws_secret_access_key=aws_credentials.secret_key,
aws_host=environ['TARGET_API_AWS_AUTH'],
aws_region=session.get_config_variable('region'),
aws_service="execute-api")
return auth
def send_creds(user: str, password: str, awsconf: dict) -> dict:
url = 'https://' + awsconf['host'] + awsconf['savecreds']
auth_headers = AWSRequestsAuth(awsconf['key'], awsconf['secret'],
awsconf['host'], awsconf['region'],
'execute-api')
content = {'UserId': user, 'Password': password}
response = requests.post(url, json=content, auth=auth_headers)
response.raise_for_status() # raise requests.HTTPException on error
response_body = json.loads(response.content, encoding='utf-8')
return response_body
def sigv4_signed_auth_headers(self, id_creds):
auth = AWSRequestsAuth(
aws_access_key=id_creds.get("Credentials").get("AccessKeyId"),
aws_secret_access_key=id_creds.get("Credentials").get("SecretKey"),
aws_token=id_creds.get("Credentials").get("SessionToken"),
aws_host=f"{StaxConfig.hostname}",
aws_region=self.aws_region,
aws_service="execute-api",
)
return auth
def send_message_to_user(connection_id, connections):
aws_auth = AWSRequestsAuth(aws_access_key=access_key_id,
aws_secret_access_key=secret_access_key,
aws_region=region,
aws_token=session_token,
aws_service="execute-api",
aws_host=aws_host)
chat_endpoint = callback_url + connection_id.replace(
"=", "") + "%3D" # encode it later
requests.post(chat_endpoint, auth=aws_auth, data=json.dumps(connections))
return {'statusCode': 200, 'headers': {'status': 'success'}}
def get_auth(host, region):
credential = credentials.get_credentials(Session())
if hasattr(credential, 'access_key'):
return AWSRequestsAuth(aws_access_key=credential.access_key,
aws_secret_access_key=credential.secret_key,
aws_token=credential.token,
aws_host=host.split(':')[0],
aws_region=region,
aws_service='es')
if os.environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"):
credential = ContainerMetadataFetcher().retrieve_uri()
return AWSRequestsAuth(aws_access_key=credential['access_key'],
aws_secret_access_key=credential['secret_key'],
aws_token=credential['token'],
aws_host=host.split(':')[0],
aws_region=region,
aws_service='es')
return None
