def update_from_cloudformation_json(cls,
original_resource, new_resource_name, cloudformation_json, region_name):
props = cloudformation_json.get('Properties', {})
for key, val in props.items():
snake_key = camel_to_snake_case(key)
lower_key = key.lower()
for candidate in [key, lower_key, snake_key]:
if hasattr(original_resource, candidate) or candidate == snake_key:
setattr(original_resource, candidate, val)
break
return original_resource
def invoke_rest_api_integration_backend(
invocation_context: ApiInvocationContext):
# define local aliases from invocation context
invocation_path = invocation_context.path_with_query_string
method = invocation_context.method
data = invocation_context.data
headers = invocation_context.headers
api_id = invocation_context.api_id
stage = invocation_context.stage
resource_path = invocation_context.resource_path
integration = invocation_context.integration
integration_response = integration.get("integrationResponses", {})
response_templates = integration_response.get("200", {}).get(
"responseTemplates", {})
# extract integration type and path parameters
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
integration_type_orig = integration.get("type") or integration.get(
"integrationType") or ""
integration_type = integration_type_orig.upper()
uri = integration.get("uri") or integration.get("integrationUri") or ""
# XXX we need replace the internal Authorization header with an Authorization header set from
# the customer, even if it's empty that's what's expected in the integration.
custom_auth_header = invocation_context.headers.pop(
HEADER_LOCALSTACK_AUTHORIZATION, "")
invocation_context.headers["Authorization"] = custom_auth_header
try:
path_params = extract_path_params(path=relative_path,
extracted_path=resource_path)
invocation_context.path_params = path_params
except Exception:
path_params = {}
if (uri.startswith("arn:aws:apigateway:")
and ":lambda:path" in uri) or uri.startswith("arn:aws:lambda"):
if integration_type == "AWS_PROXY":
return LambdaProxyIntegration().invoke(invocation_context)
elif integration_type == "AWS":
func_arn = uri
if ":lambda:path" in uri:
func_arn = (uri.split(":lambda:path")[1].split("functions/")
[1].split("/invocations")[0])
headers = helpers.create_invocation_headers(invocation_context)
invocation_context.context = helpers.get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
if invocation_context.authorizer_type:
invocation_context.context[
"authorizer"] = invocation_context.auth_context
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
# TODO: change this signature to InvocationContext as well!
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
payload,
stage,
api_id,
headers,
is_base64_encoded=invocation_context.is_data_base64_encoded,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=resource_path,
request_context=invocation_context.context,
stage_variables=invocation_context.stage_variables,
)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = (result if isinstance(result, dict) else
json.loads(str(result or "{}")))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = 200
response._content = parsed_result
update_content_length(response)
# apply custom response template
invocation_context.response = response
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
invocation_context.response.headers["Content-Length"] = str(
len(response.content or ""))
return invocation_context.response
raise Exception(
f'API Gateway integration type "{integration_type}", action "{uri}", method "{method}"'
)
elif integration_type == "AWS":
if "kinesis:action/" in uri:
if uri.endswith("kinesis:action/PutRecord"):
target = kinesis_listener.ACTION_PUT_RECORD
elif uri.endswith("kinesis:action/PutRecords"):
target = kinesis_listener.ACTION_PUT_RECORDS
elif uri.endswith("kinesis:action/ListStreams"):
target = kinesis_listener.ACTION_LIST_STREAMS
else:
LOG.info(
f"Unexpected API Gateway integration URI '{uri}' for integration type {integration_type}",
)
target = ""
try:
invocation_context.context = helpers.get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
except Exception as e:
LOG.warning("Unable to convert API Gateway payload to str", e)
raise
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(
service="kinesis", region_name=invocation_context.region_name)
headers["X-Amz-Target"] = target
result = common.make_http_request(
url=config.service_url("kineses"),
data=payload,
headers=headers,
method="POST")
# apply response template
invocation_context.response = result
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
return invocation_context.response
elif "states:action/" in uri:
action = uri.split("/")[-1]
if APPLICATION_JSON in integration.get("requestTemplates", {}):
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
payload = json.loads(payload)
else:
# XXX decoding in py3 sounds wrong, this actually might break
payload = json.loads(data.decode("utf-8"))
client = aws_stack.connect_to_service("stepfunctions")
if isinstance(payload.get("input"), dict):
payload["input"] = json.dumps(payload["input"])
# Hot fix since step functions local package responses: Unsupported Operation: 'StartSyncExecution'
method_name = (camel_to_snake_case(action)
if action != "StartSyncExecution" else
"start_execution")
try:
method = getattr(client, method_name)
except AttributeError:
msg = "Invalid step function action: %s" % method_name
LOG.error(msg)
return make_error_response(msg, 400)
result = method(**payload)
result = json_safe(
{k: result[k]
for k in result if k not in "ResponseMetadata"})
response = requests_response(
content=result,
headers=aws_stack.mock_aws_request_headers(),
)
if action == "StartSyncExecution":
# poll for the execution result and return it
result = await_sfn_execution_result(result["executionArn"])
result_status = result.get("status")
if result_status != "SUCCEEDED":
return make_error_response(
"StepFunctions execution %s failed with status '%s'" %
(result["executionArn"], result_status),
500,
)
result = json_safe(result)
response = requests_response(content=result)
# apply response templates
invocation_context.response = response
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
# response = apply_request_response_templates(
# response, response_templates, content_type=APPLICATION_JSON
# )
return response
# https://docs.aws.amazon.com/apigateway/api-reference/resource/integration/
elif ("s3:path/" in uri or "s3:action/" in uri) and method == "GET":
s3 = aws_stack.connect_to_service("s3")
uri = apply_request_parameters(
uri,
integration=integration,
path_params=path_params,
query_params=query_string_params,
)
uri_match = re.match(TARGET_REGEX_PATH_S3_URI, uri) or re.match(
TARGET_REGEX_ACTION_S3_URI, uri)
if uri_match:
bucket, object_key = uri_match.group("bucket", "object")
LOG.debug("Getting request for bucket %s object %s", bucket,
object_key)
try:
object = s3.get_object(Bucket=bucket, Key=object_key)
except s3.exceptions.NoSuchKey:
msg = "Object %s not found" % object_key
LOG.debug(msg)
return make_error_response(msg, 404)
headers = aws_stack.mock_aws_request_headers(service="s3")
if object.get("ContentType"):
headers["Content-Type"] = object["ContentType"]
# stream used so large files do not fill memory
response = request_response_stream(stream=object["Body"],
headers=headers)
return response
else:
msg = "Request URI does not match s3 specifications"
LOG.warning(msg)
return make_error_response(msg, 400)
if method == "POST":
if uri.startswith("arn:aws:apigateway:") and ":sqs:path" in uri:
template = integration["requestTemplates"][APPLICATION_JSON]
account_id, queue = uri.split("/")[-2:]
region_name = uri.split(":")[3]
if "GetQueueUrl" in template or "CreateQueue" in template:
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
new_request = f"{payload}&QueueName={queue}"
else:
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
queue_url = f"{config.get_edge_url()}/{account_id}/{queue}"
new_request = f"{payload}&QueueUrl={queue_url}"
headers = aws_stack.mock_aws_request_headers(
service="sqs", region_name=region_name)
url = urljoin(config.service_url("sqs"),
f"{TEST_AWS_ACCOUNT_ID}/{queue}")
result = common.make_http_request(url,
method="POST",
headers=headers,
data=new_request)
return result
elif uri.startswith("arn:aws:apigateway:") and ":sns:path" in uri:
invocation_context.context = helpers.get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
integration_response = SnsIntegration().invoke(
invocation_context)
return apply_request_response_templates(
integration_response,
response_templates,
content_type=APPLICATION_JSON)
raise Exception(
'API Gateway AWS integration action URI "%s", method "%s" not yet implemented'
% (uri, method))
elif integration_type == "AWS_PROXY":
if uri.startswith("arn:aws:apigateway:") and ":dynamodb:action" in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(":dynamodb:action")[1].split("&Table=")[1]
action = uri.split(":dynamodb:action")[1].split("&Table=")[0]
if "PutItem" in action and method == "PUT":
response_template = response_templates.get("application/json")
if response_template is None:
msg = "Invalid response template defined in integration response."
LOG.info("%s Existing: %s", msg, response_templates)
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template["TableName"] != table_name:
msg = "Invalid table name specified in integration response template."
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource("dynamodb")
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template["Item"].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(event_data)
return response
else:
raise Exception(
'API Gateway action uri "%s", integration type %s not yet implemented'
% (uri, integration_type))
elif integration_type in ["HTTP_PROXY", "HTTP"]:
if ":servicediscovery:" in uri:
# check if this is a servicediscovery integration URI
client = aws_stack.connect_to_service("servicediscovery")
service_id = uri.split("/")[-1]
instances = client.list_instances(
ServiceId=service_id)["Instances"]
instance = (instances or [None])[0]
if instance and instance.get("Id"):
uri = "http://%s/%s" % (instance["Id"],
invocation_path.lstrip("/"))
# apply custom request template
invocation_context.context = helpers.get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
if isinstance(payload, dict):
payload = json.dumps(payload)
uri = apply_request_parameters(
uri,
integration=integration,
path_params=path_params,
query_params=query_string_params,
)
result = requests.request(method=method,
url=uri,
data=payload,
headers=headers)
# apply custom response template
invocation_context.response = result
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
return invocation_context.response
elif integration_type == "MOCK":
mock_integration = MockIntegration()
return mock_integration.invoke(invocation_context)
if method == "OPTIONS":
# fall back to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
raise Exception(
'API Gateway integration type "%s", method "%s", URI "%s" not yet implemented'
% (integration_type, method, uri))
def generate_service_api(output, service: ServiceModel, doc=True):
service_name = service.service_name.replace("-", "_")
class_name = service_name + "_api"
class_name = snake_to_camel_case(class_name)
output.write(f"class {class_name}:\n")
output.write("\n")
output.write(f' service = "{service.service_name}"\n')
output.write(f' version = "{service.api_version}"\n')
for op_name in service.operation_names:
operation: OperationModel = service.operation_model(op_name)
fn_name = camel_to_snake_case(op_name)
if operation.output_shape:
output_shape = operation.output_shape.name
else:
output_shape = "None"
output.write("\n")
parameters = OrderedDict()
param_shapes = OrderedDict()
input_shape = operation.input_shape
if input_shape is not None:
members = list(input_shape.members)
for m in input_shape.required_members:
members.remove(m)
m_shape = input_shape.members[m]
parameters[xform_name(m)] = m_shape.name
param_shapes[xform_name(m)] = m_shape
for m in members:
m_shape = input_shape.members[m]
param_shapes[xform_name(m)] = m_shape
parameters[xform_name(m)] = f"{m_shape.name} = None"
if any(map(is_bad_param_name, parameters.keys())):
# if we cannot render the parameter name, don't expand the parameters in the handler
param_list = f"request: {input_shape.name}" if input_shape else ""
output.write(f' @handler("{operation.name}", expand=False)\n')
else:
param_list = ", ".join([f"{k}: {v}" for k, v in parameters.items()])
output.write(f' @handler("{operation.name}")\n')
output.write(
f" def {fn_name}(self, context: RequestContext, {param_list}) -> {output_shape}:\n"
)
# convert html documentation to rst and print it into to the signature
if doc:
html = operation.documentation
import pypandoc
doc = pypandoc.convert_text(html, "rst", format="html")
output.write(' """')
output.write(f"{doc.strip()}\n")
output.write("\n")
# parameters
for param_name, shape in param_shapes.items():
# FIXME: this doesn't work properly
pdoc = pypandoc.convert_text(shape.documentation, "rst", format="html")
pdoc = pdoc.strip().split(".")[0] + "."
output.write(f":param {param_name}: {pdoc}\n")
# return value
if operation.output_shape:
output.write(f":returns: {operation.output_shape.name}\n")
# errors
for error in operation.error_shapes:
output.write(f":raises {error.name}:\n")
output.write(' """\n')
output.write(" raise NotImplementedError\n")
def invoke_rest_api_integration_backend(
invocation_context: ApiInvocationContext, integration: Dict):
# define local aliases from invocation context
invocation_path = invocation_context.path_with_query_string
method = invocation_context.method
path = invocation_context.path
data = invocation_context.data
headers = invocation_context.headers
api_id = invocation_context.api_id
stage = invocation_context.stage
resource_path = invocation_context.resource_path
response_templates = invocation_context.response_templates
# extract integration type and path parameters
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
integration_type_orig = integration.get("type") or integration.get(
"integrationType") or ""
integration_type = integration_type_orig.upper()
uri = integration.get("uri") or integration.get("integrationUri") or ""
try:
path_params = extract_path_params(path=relative_path,
extracted_path=resource_path)
except Exception:
path_params = {}
if (uri.startswith("arn:aws:apigateway:")
and ":lambda:path" in uri) or uri.startswith("arn:aws:lambda"):
if integration_type in ["AWS", "AWS_PROXY"]:
func_arn = uri
if ":lambda:path" in uri:
func_arn = (uri.split(":lambda:path")[1].split("functions/")
[1].split("/invocations")[0])
# apply custom request template
data_str = data
is_base64_encoded = False
try:
data_str = json.dumps(data) if isinstance(
data, (dict, list)) else to_str(data)
data_str = apply_template(
integration,
"request",
data_str,
path_params=path_params,
query_params=query_string_params,
headers=headers,
)
except UnicodeDecodeError:
data_str = base64.b64encode(data_str)
is_base64_encoded = True
except Exception as e:
LOG.warning(
"Unable to convert API Gateway payload to str: %s" % (e))
pass
# Sample request context:
# https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-create-api-as-simple-proxy-for-lambda.html#api-gateway-create-api-as-simple-proxy-for-lambda-test
request_context = get_lambda_event_request_context(
invocation_context)
stage_variables = (get_stage_variables(api_id, stage)
if not is_test_invoke_method(method, path) else
None)
# TODO: change this signature to InvocationContext as well!
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
data_str,
stage,
api_id,
headers,
is_base64_encoded=is_base64_encoded,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=resource_path,
request_context=request_context,
event_context=invocation_context.context,
stage_variables=stage_variables,
)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = (result if isinstance(result, dict) else
json.loads(str(result or "{}")))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get(
"statusCode", 200))
parsed_headers = parsed_result.get("headers", {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
result_body = parsed_result.get("body")
if isinstance(result_body, dict):
response._content = json.dumps(result_body)
else:
body_bytes = to_bytes(to_str(result_body or ""))
if parsed_result.get("isBase64Encoded", False):
body_bytes = base64.b64decode(body_bytes)
response._content = body_bytes
except Exception as e:
LOG.warning("Couldn't set Lambda response content: %s" % e)
response._content = "{}"
update_content_length(response)
response.multi_value_headers = parsed_result.get(
"multiValueHeaders") or {}
# apply custom response template
response._content = apply_template(integration, "response",
response._content)
response.headers["Content-Length"] = str(
len(response.content or ""))
return response
raise Exception(
'API Gateway integration type "%s", action "%s", method "%s" invalid or not yet implemented'
% (integration_type, uri, method))
elif integration_type == "AWS":
if "kinesis:action/" in uri:
if uri.endswith("kinesis:action/PutRecord"):
target = kinesis_listener.ACTION_PUT_RECORD
elif uri.endswith("kinesis:action/PutRecords"):
target = kinesis_listener.ACTION_PUT_RECORDS
elif uri.endswith("kinesis:action/ListStreams"):
target = kinesis_listener.ACTION_LIST_STREAMS
else:
LOG.info(
"Unexpected API Gateway integration URI '%s' for integration type %s",
uri,
integration_type,
)
target = ""
# apply request templates
new_data = apply_request_response_templates(
data,
integration.get("requestTemplates"),
content_type=APPLICATION_JSON)
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(service="kinesis")
headers["X-Amz-Target"] = target
result = common.make_http_request(url=config.TEST_KINESIS_URL,
method="POST",
data=new_data,
headers=headers)
# apply response template
result = apply_request_response_templates(
result, response_templates, content_type=APPLICATION_JSON)
return result
elif "states:action/" in uri:
action = uri.split("/")[-1]
payload = {}
if APPLICATION_JSON in integration.get("requestTemplates", {}):
payload = apply_request_response_templates(
data,
integration.get("requestTemplates"),
content_type=APPLICATION_JSON,
as_json=True,
)
else:
payload = json.loads(data.decode("utf-8"))
client = aws_stack.connect_to_service("stepfunctions")
# Hot fix since step functions local package responses: Unsupported Operation: 'StartSyncExecution'
method_name = (camel_to_snake_case(action)
if action != "StartSyncExecution" else
"start_execution")
try:
method = getattr(client, method_name)
except AttributeError:
msg = "Invalid step function action: %s" % method_name
LOG.error(msg)
return make_error_response(msg, 400)
result = method(**payload, )
result = json_safe(
{k: result[k]
for k in result if k not in "ResponseMetadata"})
response = requests_response(
content=result,
headers=aws_stack.mock_aws_request_headers(),
)
if action == "StartSyncExecution":
# poll for the execution result and return it
result = await_sfn_execution_result(result["executionArn"])
result_status = result.get("status")
if result_status != "SUCCEEDED":
return make_error_response(
"StepFunctions execution %s failed with status '%s'" %
(result["executionArn"], result_status),
500,
)
result = json_safe(result)
response = requests_response(content=result)
# apply response templates
response = apply_request_response_templates(
response, response_templates, content_type=APPLICATION_JSON)
return response
elif "s3:path/" in uri and method == "GET":
s3 = aws_stack.connect_to_service("s3")
uri_match = re.match(TARGET_REGEX_S3_URI, uri)
if uri_match:
bucket, object_key = uri_match.group("bucket", "object")
LOG.debug("Getting request for bucket %s object %s", bucket,
object_key)
try:
object = s3.get_object(Bucket=bucket, Key=object_key)
except s3.exceptions.NoSuchKey:
msg = "Object %s not found" % object_key
LOG.debug(msg)
return make_error_response(msg, 404)
headers = aws_stack.mock_aws_request_headers(service="s3")
if object.get("ContentType"):
headers["Content-Type"] = object["ContentType"]
# stream used so large files do not fill memory
response = request_response_stream(stream=object["Body"],
headers=headers)
return response
else:
msg = "Request URI does not match s3 specifications"
LOG.warning(msg)
return make_error_response(msg, 400)
if method == "POST":
if uri.startswith("arn:aws:apigateway:") and ":sqs:path" in uri:
template = integration["requestTemplates"][APPLICATION_JSON]
account_id, queue = uri.split("/")[-2:]
region_name = uri.split(":")[3]
new_request = "%s&QueueName=%s" % (
aws_stack.render_velocity_template(template, data),
queue,
)
headers = aws_stack.mock_aws_request_headers(
service="sqs", region_name=region_name)
url = urljoin(config.TEST_SQS_URL,
"%s/%s" % (TEST_AWS_ACCOUNT_ID, queue))
result = common.make_http_request(url,
method="POST",
headers=headers,
data=new_request)
return result
raise Exception(
'API Gateway AWS integration action URI "%s", method "%s" not yet implemented'
% (uri, method))
elif integration_type == "AWS_PROXY":
if uri.startswith("arn:aws:apigateway:") and ":dynamodb:action" in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(":dynamodb:action")[1].split("&Table=")[1]
action = uri.split(":dynamodb:action")[1].split("&Table=")[0]
if "PutItem" in action and method == "PUT":
response_template = response_templates.get("application/json")
if response_template is None:
msg = "Invalid response template defined in integration response."
LOG.info("%s Existing: %s" % (msg, response_templates))
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template["TableName"] != table_name:
msg = "Invalid table name specified in integration response template."
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource("dynamodb")
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template["Item"].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(event_data)
return response
else:
raise Exception(
'API Gateway action uri "%s", integration type %s not yet implemented'
% (uri, integration_type))
elif integration_type in ["HTTP_PROXY", "HTTP"]:
if ":servicediscovery:" in uri:
# check if this is a servicediscovery integration URI
client = aws_stack.connect_to_service("servicediscovery")
service_id = uri.split("/")[-1]
instances = client.list_instances(
ServiceId=service_id)["Instances"]
instance = (instances or [None])[0]
if instance and instance.get("Id"):
uri = "http://%s/%s" % (instance["Id"],
invocation_path.lstrip("/"))
# apply custom request template
data = apply_template(integration, "request", data)
if isinstance(data, dict):
data = json.dumps(data)
uri = apply_request_parameter(uri,
integration=integration,
path_params=path_params)
result = requests.request(method=method,
url=uri,
data=data,
headers=headers)
# apply custom response template
result = apply_template(integration, "response", result)
return result
elif integration_type == "MOCK":
# return empty response - details filled in via responseParameters above...
return requests_response({})
if method == "OPTIONS":
# fall back to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
raise Exception(
'API Gateway integration type "%s", method "%s", URI "%s" not yet implemented'
% (integration_type, method, uri))
