def signed_request(method,
url,
data=None,
params=None,
headers=None,
service=None):
request = AWSRequest(method=method,
url=url,
data=data,
params=params,
headers=headers)
session = boto3.Session()
credentials = session.get_credentials()
try:
frozen_creds = credentials.get_frozen_credentials()
except AttributeError:
print(
"Could not find valid IAM credentials in any the following locations:\n"
)
print(
"env, assume-role, assume-role-with-web-identity, sso, shared-credential-file, custom-process, "
"config-file, ec2-credentials-file, boto-config, container-role, iam-role\n"
)
print(
"Go to https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html for more "
"details on configuring your IAM credentials.")
return request
SigV4Auth(frozen_creds, service,
boto3.Session().region_name).add_auth(request)
return requests.request(method=method,
url=url,
headers=dict(request.headers),
data=data)
def create_token_aws(project_id: str, pool_id: str, provider_id: str) -> None:
# Prepare a GetCallerIdentity request.
request = AWSRequest(
method="POST",
url=
"https://sts.amazonaws.com/?Action=GetCallerIdentity&Version=2011-06-15",
headers={
"Host":
"sts.amazonaws.com",
"x-goog-cloud-target-resource":
f"//iam.googleapis.com/projects/{project_id}/locations/global/workloadIdentityPools/{pool_id}/providers/{provider_id}"
})
# Set the session credentials and Sign the request.
# get_credentials loads the required credentials as environment variables.
# Refer:
# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
SigV4Auth(boto3.Session().get_credentials(), "sts",
"us-east-1").add_auth(request)
# Create token from signed request.
token = {"url": request.url, "method": request.method, "headers": []}
for key, value in request.headers.items():
token["headers"].append({"key": key, "value": value})
# The token lets workload identity federation verify the identity without revealing the AWS secret access key.
print("Token:\n%s" % json.dumps(token, indent=2, sort_keys=True))
print("URL encoded token:\n%s" % urllib.parse.quote(json.dumps(token)))
def _get_aws_request(self,
method,
url,
*,
data=None,
params=None,
headers=None,
service=NEPTUNE_SERVICE_NAME):
req = AWSRequest(method=method,
url=url,
data=data,
params=params,
headers=headers)
if self.iam_enabled:
credentials = self._session.get_credentials()
try:
frozen_creds = credentials.get_frozen_credentials()
except AttributeError:
print(
"Could not find valid IAM credentials in any the following locations:\n"
)
print(
"env, assume-role, assume-role-with-web-identity, sso, shared-credential-file, custom-process, "
"config-file, ec2-credentials-file, boto-config, container-role, iam-role\n"
)
print(
"Go to https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html for more "
"details on configuring your IAM credentials.")
return req
SigV4Auth(frozen_creds, service, self.region).add_auth(req)
prepared_iam_req = req.prepare()
return prepared_iam_req
else:
return req
def start(self):
try:
self.logger.info('about to get new credentials')
self.credentials = botocore.credentials.search_iam_role()
self.sigV4auth = SigV4Auth(self.credentials, self.service,
self.region)
self.logger.info('finish getting new credentials')
if self.role:
role_metadata = botocore.credentials._search_md()[self.role]
else:
role_metadata = botocore.credentials._search_md().values()[0]
utc_expiration = dateutil.parser.parse(role_metadata['Expiration'])
t_delta = utc_expiration - datetime.datetime.now(
tzutc()) - self.before
self.logger.info(
'next time to get credentials would be {} seconds later'.
format(t_delta.total_seconds()))
self._timeout = self.io_loop.add_timeout(t_delta, self.start)
except:
self.logger.exception(
'Exception found, would retry in {} seconds'.format(
self.fail_retry.total_seconds()))
self._timeout = self.io_loop.add_timeout(self.fail_retry,
self.start)
def create_signed_headers(url, payload):
host_segments = urlparse(url).netloc.split(".")
service = host_segments[0]
region = host_segments[1]
request = AWSRequest(method="POST", url=url, data=payload)
SigV4Auth(CREDS, service, region).add_auth(request)
return dict(request.headers.items())
def __init__(
self,
host: str,
region_name: Optional[str] = None,
signer: Optional["botocore.auth.BaseSigner"] = None,
request_creator: Optional[Callable[[Dict[
str, Any]], "botocore.awsrequest.AWSRequest"]] = None,
credentials: Optional["botocore.credentials.Credentials"] = None,
session: Optional["botocore.session.Session"] = None,
) -> None:
"""Initialize itself, saving the found credentials used
to sign the headers later.
if no credentials are found, then a NoCredentialsError is raised.
"""
from botocore.auth import SigV4Auth
from botocore.awsrequest import create_request_object
from botocore.session import get_session
self._host = host.replace("appsync-realtime-api", "appsync-api")
self._session = session if session else get_session()
self._credentials = (credentials if credentials else
self._session.get_credentials())
self._service_name = "appsync"
self._region_name = region_name or self._detect_region_name()
self._signer = (signer if signer else SigV4Auth(
self._credentials, self._service_name, self._region_name))
self._request_creator = (request_creator
if request_creator else create_request_object)
def _prepare_request(self,
method,
url,
*,
data=None,
params=None,
headers=None,
service=NEPTUNE_SERVICE_NAME):
self._ensure_http_session()
request = requests.Request(method=method,
url=url,
data=data,
params=params,
headers=headers,
auth=self._auth)
if self._session is not None:
credentials = self._session.get_credentials()
frozen_creds = credentials.get_frozen_credentials()
req = AWSRequest(method=method,
url=url,
data=data,
params=params,
headers=headers)
SigV4Auth(frozen_creds, service, self.region).add_auth(req)
prepared_iam_req = req.prepare()
request.headers = dict(prepared_iam_req.headers)
return request.prepare()
def _aws_auth_header(credentials, server_nonce, sts_host):
"""Signature Version 4 Signing Process to construct the authorization header
"""
region = _get_region(sts_host)
request_parameters = 'Action=GetCallerIdentity&Version=2011-06-15'
encoded_nonce = standard_b64encode(server_nonce).decode('utf8')
request_headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': str(len(request_parameters)),
'Host': sts_host,
'X-MongoDB-Server-Nonce': encoded_nonce,
'X-MongoDB-GS2-CB-Flag': 'n',
}
request = AWSRequest(method="POST", url="/", data=request_parameters,
headers=request_headers)
boto_creds = Credentials(credentials.username, credentials.password,
token=credentials.token)
auth = SigV4Auth(boto_creds, "sts", region)
auth.add_auth(request)
final = {
'a': request.headers['Authorization'],
'd': request.headers['X-Amz-Date']
}
if credentials.token:
final['t'] = credentials.token
return final
def make_request(endpoint, data, method='GET'):
"""
This function handles a HTTP request to a given elasticsearch domain endpoint and method.
:param endpoint: The elasticsearch domain endpoint
:param data:
:param method: the type of HTTP method to use. eg: 'GET POST DELETE etc'
:return: The response of the request
"""
host = endpoint
endpoint = '{0}/_bulk'.format(host)
region = endpoint.split('.')[1]
service = endpoint.split('.')[2]
credentials = boto3.session.Session().get_credentials()
request = AWSRequest(method=method,
url='https://{0}'.format(endpoint),
data=data)
SigV4Auth(credentials, service, region).add_auth(request)
headers = dict(request.headers.items())
opener = urllib2.build_opener(urllib2.HTTPHandler)
request = urllib2.Request('https://{0}'.format(endpoint), request.data)
request.add_header('Host', host)
request.add_header('Content-Type', 'application/json')
request.add_header('X-Amz-Date', headers['X-Amz-Date'])
request.add_header('X-Amz-Security-Token', headers['X-Amz-Security-Token'])
request.add_header('Authorization', headers['Authorization'])
request.get_method = lambda: method
print(request.data)
return opener.open(request).read()
def __init__(self, boto3_session=None, service_name='execute-api'):
self.boto3_session = boto3_session or boto3.Session()
self.sigv4 = SigV4Auth(
credentials=self.boto3_session.get_credentials(),
service_name=service_name,
region_name=self.boto3_session.region_name,
)
def post_data_to_es(payload,
region,
creds,
host,
path,
method='POST',
proto='https://'):
'''Post data to ES endpoint with SigV4 signed http headers'''
"""
Low-level POST data to Amazon Elasticsearch Service generating a Sigv4 signed request
:param payload:
:param region:
:param creds:
:param host:
:param path:
:param method:
:param proto:
:return:
"""
req = AWSRequest(method=method,
url=proto + host + urllib.quote(path),
data=payload,
headers={'Host': host})
SigV4Auth(creds, 'es', region).add_auth(req)
http_session = BotocoreHTTPSession()
res = http_session.send(req.prepare())
if 200 <= res.status_code <= 299:
return res._content
else:
raise ESException(res.status_code, res._content)
def sign_request_headers(self,
method: str,
path: str,
headers: dict,
body: str,
params: object = None) -> HTTPHeaders:
"""
Construct the request headers, including the signature
:param method: The HTTP method
:param path: The URL path
:param headers: The request headers
:param body: The request body
:param params: The query parameters
:return:
"""
request = AWSRequest(method=method,
url=urljoin(self.url, path),
auth_path=path,
data=body,
params=params,
headers=headers)
SigV4Auth(self.boto_session.get_credentials(), SERVICE_NAME,
REGION_NAME).add_auth(request)
return request.headers
def lambda_handler(request):
"""
Sign the request and forward it to S3.
"""
if not (request.method == 'POST' and 'select' in request.args):
return requests.codes.bad_request, 'Not an S3 select', {
'content-type': 'text/plain'
}
bucket, key = request.pathParameters['proxy'].split('/', 1)
host = f'{bucket}.s3.amazonaws.com'
# Make an unsigned HEAD request to test anonymous access.
object_url = f'https://{host}/{key}'
head_response = session.head(object_url)
if not head_response.ok:
return requests.codes.forbidden, 'Not allowed', {
'content-type': 'text/plain'
}
# Sign the full S3 select request.
url = f'{object_url}?{urlencode(request.args)}'
headers = {
k: v
for k, v in request.headers.items() if k in REQUEST_HEADERS_TO_FORWARD
}
headers['host'] = host
aws_request = AWSRequest(method=request.method,
url=url,
data=request.data,
headers={
k: v
for k, v in headers.items()
if k in REQUEST_HEADERS_TO_SIGN
})
credentials = Session().get_credentials()
auth = SigV4Auth(credentials, SERVICE, REGION)
auth.add_auth(aws_request)
headers.update(aws_request.headers)
response = session.post(
url=url,
data=request.data, # Forward the POST data.
headers=headers,
)
response_headers = {
k: v
for k, v in response.headers.items()
if k in RESPONSE_HEADERS_TO_FORWARD
}
# Add a default content type to prevent API Gateway from setting it to application/json.
response_headers.setdefault('content-type', 'application/octet-stream')
return response.status_code, response.content, response_headers
def _validate_signature(self, headers, raw_input):
auth_header = headers["Authorization"]
signed_headers_start = auth_header.find("SignedHeaders")
signed_headers = auth_header[signed_headers_start:auth_header.
find(",", signed_headers_start)]
signed_headers_dict = get_dict_subset(headers, signed_headers)
request = AWSRequest(method="POST",
url="/",
data=raw_input,
headers=signed_headers_dict)
# SigV4Auth assumes this header exists even though it is not required by the algorithm
request.context['timestamp'] = headers['X-Amz-Date']
region_start = auth_header.find("Credential=access/") + len(
"Credential=access/YYYYMMDD/")
region = auth_header[region_start:auth_header.find("/", region_start)]
credentials = Credentials("access", "secret")
auth = SigV4Auth(credentials, "kms", region)
string_to_sign = auth.string_to_sign(request,
auth.canonical_request(request))
expected_signature = auth.signature(string_to_sign, request)
signature_headers_start = auth_header.find("Signature=") + len(
"Signature=")
actual_signature = auth_header[signature_headers_start:]
return expected_signature == actual_signature
def invoke(self, runtime_name, runtime_memory, payload):
"""
Invoke lambda function asynchronously
@param runtime_name: name of the runtime
@param runtime_memory: memory of the runtime in MB
@param payload: invoke dict payload
@return: invocation ID
"""
function_name = self._format_function_name(runtime_name, runtime_memory)
headers = {'Host': self.host, 'X-Amz-Invocation-Type': 'Event', 'User-Agent': self.user_agent}
url = f'https://{self.host}/2015-03-31/functions/{function_name}/invocations'
request = AWSRequest(method="POST", url=url, data=json.dumps(payload, default=str), headers=headers)
SigV4Auth(self.credentials, "lambda", self.region_name).add_auth(request)
invoked = False
while not invoked:
try:
r = self.session.send(request.prepare())
invoked = True
except Exception:
pass
if r.status_code == 202:
return r.headers['x-amzn-RequestId']
elif r.status_code == 401:
logger.debug(r.text)
raise Exception('Unauthorized - Invalid API Key')
elif r.status_code == 404:
logger.debug(r.text)
raise Exception('Lithops Runtime: {} not deployed'.format(runtime_name))
else:
logger.debug(r.text)
raise Exception('Error {}: {}'.format(r.status_code, r.text))
def post_data_to_es(payload,
region,
creds,
host,
path,
method='POST',
proto='https://'):
print("URL:{}".format(proto + host + path))
req = AWSRequest(method=method,
url=proto + host + path,
data=payload,
headers={
'Host': host,
'Content-Type': 'application/json'
})
SigV4Auth(creds, 'es', region).add_auth(req)
http_session = BotocoreHTTPSession()
res = http_session.send(req.prepare())
print("STATUS_CODE:{}".format(res.status_code))
print("CONTENT:{}".format(res._content))
print("ALL:{}".format(res))
if res.status_code >= 200 and res.status_code <= 299:
return res._content
else:
raise ES_Exception(res.status_code, res._content)
def __init__(self, region: str, access_key: Optional[str],
secret_key: Optional[str],
loop: Optional[asyncio.AbstractEventLoop]):
self.region = region
self.base_url = self._base_url_template.format(
service_name=self._service_name,
region=self.region,
)
self.__signer = SigV4Auth(credentials=get_credentials(
access_key, secret_key),
service_name=self._service_name,
region_name=self.region)
if not loop:
loop = asyncio.get_event_loop()
self.loop = loop
ssl_context = ssl.create_default_context(cafile=certifi.where())
connector = aiohttp.TCPConnector(ssl=ssl_context, loop=self.loop)
self.session = aiohttp.ClientSession(connector=connector,
loop=self.loop,
json_serialize=json.dumps)
self.set_current(self)
def get_auth_instance(self,
signing_name,
region_name,
signature_version=None,
**kwargs):
# create new signer for API Gateway
# addresses `Credential should be scoped to correct service: 'execute-api'` error
return SigV4Auth(self._credentials, "execute-api", region_name)
def __sign_request(self, request):
"""Sign request to AWS with SigV4Auth."""
aws_request = AWSRequest(method=request.method,
url=request.url,
data=request.body)
signer = SigV4Auth(self._credentials, 'lambda', self.region)
signer.add_auth(aws_request)
request.headers.update(dict(aws_request.headers.items()))
def getSignedRequest():
signedRequest = AWSRequest(method=r.method.upper(),
headers=r.headers,
url=r.url,
data=r.body)
SigV4Auth(self.credentials, 'execute-api',
self.region_name).add_auth(signedRequest)
return signedRequest
def __init__(self) -> None:
current_session = Session()
region = current_session.get_config_variable('region')
creds = current_session.get_credentials()
self.signer = SigV4Auth(creds, 'execute-api', region)
analysis_api_fqdn = os.environ.get('ANALYSIS_API_FQDN')
analysis_api_path = os.environ.get('ANALYSIS_API_PATH')
self.url = 'https://' + analysis_api_fqdn + '/' + analysis_api_path
def send_to_es(self, path, method="GET", payload={}):
"""Low-level POST data to Amazon Elasticsearch Service generating a Sigv4 signed request
Args:
path (str): path to send to ES
method (str, optional): HTTP method default:GET
payload (dict, optional): additional payload used during POST or PUT
Returns:
dict: json answer converted in dict
Raises:
#: Error during ES communication
ES_Exception: Description
"""
if not path.startswith("/"):
path = "/" + path
es_region = self.cfg["es_endpoint"].split(".")[1]
headers = {
"Host": self.cfg["es_endpoint"],
"Content-Type": "application/json"
}
# send to ES with exponential backoff
retries = 0
while retries < int(self.cfg["es_max_retry"]):
if retries > 0:
seconds = (2**retries) * .1
time.sleep(seconds)
req = AWSRequest(
method=method,
url="https://{}{}".format(
self.cfg["es_endpoint"], quote(path)),
data=json.dumps(payload),
params={"format": "json"},
headers=headers)
credential_resolver = create_credential_resolver(get_session())
credentials = credential_resolver.load_credentials()
SigV4Auth(credentials, 'es', es_region).add_auth(req)
try:
preq = req.prepare()
session = Session()
res = session.send(preq)
if res.status_code >= 200 and res.status_code <= 299:
return json.loads(res.content)
else:
raise ES_Exception(res.status_code, res._content)
except ES_Exception as e:
if (e.status_code >= 500) and (e.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
def sign(request, *, aws_session, region, service):
aws_request = AWSRequest(
method=request.method.upper(),
url=to_canonical_url(request.url),
data=request.body,
)
credentials = aws_session.get_credentials()
SigV4Auth(credentials, service, region).add_auth(request)
request.headers.update(**aws_request.headers.items())
def post_data_to_es(payload, region, creds, host, path, method='POST', proto='https://'):
'''Post data to ES endpoint with SigV4 signed http headers'''
req = AWSRequest(method=method, url=proto + host + urllib.quote(path), data=payload, headers={'Host': host, 'Content-Type' : 'application/json'})
SigV4Auth(creds, 'es', region).add_auth(req)
http_session = BotocoreHTTPSession()
res = http_session.send(req.prepare())
if res.status_code >= 200 and res.status_code <= 299:
return res._content
else:
raise ES_Exception(res.status_code, res._content)
def text(self, input_text, session_attributes=None):
"""Input text will be passed to your lex bot"""
url = self.url + 'text'
payload = json.dumps({
"inputText": input_text,
"sessionAttributes": session_attributes
})
request = AWSRequest(method="POST", url=url, data=payload)
SigV4Auth(self.creds, 'lex', self.region).add_auth(request)
return self.session.send(request.prepare()).json()
def __call__(self, r):
url = urlparse(r.url)
path = url.path or '/'
qs = url.query and '?%s' % url.query or ''
safe_url = url.scheme + '://' + url.netloc.split(':')[0] + path + qs
request = AWSRequest(
method=r.method.upper(), url=safe_url, data=r.body)
SigV4Auth(
self.credentials, self.service, self.region).add_auth(request)
r.headers.update(dict(request.headers.items()))
return r
def post_data_to_opensearch(payload, region, creds, host, path, method='POST', proto='https://'):
'''Post data to OpenSearch endpoint with SigV4 signed http headers'''
req = AWSRequest(method=method, url=proto + host +
quote(path), data=payload, headers={'Host': host, 'Content-Type': 'application/json'})
# SigV4Auth may be expecting 'es' but need to swap with 'os' or 'OpenSearch'
SigV4Auth(creds, 'es', region).add_auth(req)
http_session = BotocoreHTTPSession()
res = http_session.send(req.prepare())
if res.status_code >= 200 and res.status_code <= 299:
return res._content
else:
raise Searchable_Exception(res.status_code, res._content)
def __call__(self, r):
url = urlparse(r.url)
path = url.path or '/'
querystring = ''
if url.query:
querystring = '?' + urlencode(parse_qs(url.query, keep_blank_values=True), doseq=True)
headers = {k.lower():v for k,v in r.headers.items()}
location = headers.get('host') or url.netloc
safe_url = url.scheme + '://' + location + path + querystring
request = AWSRequest(method=r.method.upper(), url=safe_url, data=r.body)
SigV4Auth(self.credentials, self.service, self.region).add_auth(request)
r.headers.update(dict(request.headers.items()))
return r
def __init__(self,
access_key,
secret_key,
service,
region='ap-southeast-2'):
self.region = region
self.access_key = access_key
self.secret_key = secret_key
self.credentials = botocore.credentials.Credentials(
self.access_key, self.secret_key)
self.service = service
self.sigV4auth = SigV4Auth(self.credentials, self.service, self.region)
def __call__(
self,
request: requests.PreparedRequest) -> requests.PreparedRequest:
"""See requests framework."""
# The code does not implement the signature v4 protocol. It reuses parts of
# botocore to do so. Botocore does not provide a generic interface to sign
# a given request. So the following code create a temporary AWSRequest object
# with the same parameters as the user request, sign it and then extract
# the signature (i.e: the headers) that is re-injected into the user request.
session = self.session
if self.role is not None:
session = self.session.assume_role(self.role, "iamauthsession")
credentials = session.session.get_credentials().get_frozen_credentials(
)
# Split back the url in order to be able to call AWSRequest
aws_headers = dict(request.headers)
key_to_delete = []
for key in aws_headers:
if key.lower() in ("accept", "accept-encoding", "connection"):
key_to_delete.append(key)
for key in key_to_delete:
del aws_headers[key]
parsed_url = urllib.parse.urlparse(request.url)
aws_params = {
str(k): ",".join(v)
for k, v in urllib.parse.parse_qs(str(parsed_url.query)).items()
}
aws_url = urllib.parse.urlunparse((
str(parsed_url.scheme),
str(parsed_url.netloc),
str(parsed_url.path),
str(parsed_url.params),
"",
"",
))
# Compute the headers for the request
aws_request = AWSRequest(method=request.method,
url=aws_url,
params=aws_params,
headers=aws_headers)
SigV4Auth(credentials, "execute-api",
self.region).add_auth(aws_request)
# Update request headers
request.headers.update(aws_request.headers)
return request
