def is_s3_form_data(data_bytes): if to_bytes("key=") in data_bytes: return True if ( to_bytes("Content-Disposition: form-data") in data_bytes and to_bytes('name="key"') in data_bytes ): return True return False
def call_moto(context: RequestContext, include_response_metadata=False) -> ServiceResponse: """ Call moto with the given request context and receive a parsed ServiceResponse. :param context: the request context :param include_response_metadata: whether to include botocore's "ResponseMetadata" attribute :return: a serialized AWS ServiceResponse (same as boto3 would return) """ status, headers, content = dispatch_to_moto(context) operation_model = context.operation response_dict = { # this is what botocore.endpoint.convert_to_response_dict normally does "headers": dict(headers.items()), # boto doesn't like werkzeug headers "status_code": status, "body": to_bytes(content), "context": { "operation_name": operation_model.name, }, } parser = create_parser(context.service.protocol) response = parser.parse(response_dict, operation_model.output_shape) if status >= 301: error = response["Error"] raise CommonServiceException( code=error.get("Code", "UnknownError"), status_code=status, message=error.get("Message", ""), ) if not include_response_metadata: response.pop("ResponseMetadata", None) return response
def get_parameters_for_import( self, context: RequestContext, key_id: KeyIdType, wrapping_algorithm: AlgorithmSpec, wrapping_key_spec: WrappingKeySpec, ) -> GetParametersForImportResponse: key = _generate_data_key_pair({"KeySpec": wrapping_key_spec}, create_cipher=False, add_to_keys=False) import_token = short_uid() import_state = KeyImportState( key_id=key_id, import_token=import_token, private_key=key["PrivateKeyPlaintext"], public_key=key["PublicKey"], wrapping_algo=wrapping_algorithm, key_obj=key["_key_"], ) KMSBackend.get().imports[import_token] = import_state expiry_date = datetime.datetime.now() + datetime.timedelta(days=100) return GetParametersForImportResponse( KeyId=key_id, ImportToken=to_bytes(import_state.import_token), PublicKey=import_state.public_key, ParametersValidTo=expiry_date, )
def _adjust_partition(self, source, static_partition: str = None): # Call this function recursively if we get a dictionary or a list if isinstance(source, dict): result = {} for k, v in source.items(): result[k] = self._adjust_partition(v, static_partition) return result if isinstance(source, list): result = [] for v in source: result.append(self._adjust_partition(v, static_partition)) return result elif isinstance(source, bytes): try: decoded = unquote(to_str(source)) adjusted = self._adjust_partition(decoded, static_partition) return to_bytes(adjusted) except UnicodeDecodeError: # If the body can't be decoded to a string, we return the initial source return source elif not isinstance(source, str): # Ignore any other types return source return self.arn_regex.sub( lambda m: self._adjust_match(m, static_partition), source)
def return_response(self, method, path, data, headers, response): api = headers.get(HEADER_TARGET_API) or "" if is_trace_logging_enabled(headers): # print response trace for debugging, if enabled if api and api != API_UNKNOWN: LOG.debug( 'OUT(%s): "%s %s" - status: %s - response headers: %s - response: %s', api, method, path, response.status_code, dict(response.headers), response.content, ) if ( response._content and headers.get("Accept-Encoding") == "gzip" and api not in SKIP_GZIP_APIS and not response.headers.pop(HEADER_SKIP_RESPONSE_ZIPPING, None) ): # services may decide to set HEADER_SKIP_RESPONSE_ZIPPING in the response, to skip result transformations response._content = gzip.compress(to_bytes(response._content)) response.headers["Content-Length"] = str(len(response._content)) response.headers["Content-Encoding"] = "gzip"
def get_recordable_data(request_data): if request_data or request_data in ["", b""]: try: request_data = to_bytes(request_data) except Exception as ex: LOG.warning("Unable to call to_bytes: %s", ex) request_data = to_str(base64.b64encode(request_data)) return request_data
def _post_process_response_headers(response: RoutingResponse) -> None: """Adjust potential content lengths and checksums after modifying the response.""" if response.headers and response.content: if "Content-Length" in response.headers: response.headers["Content-Length"] = str( len(to_bytes(response.content))) if "x-amz-crc32" in response.headers: response.headers["x-amz-crc32"] = calculate_crc32( response.content)
def forward_request(self, method, path, data, headers): # send request to target url = f"{base_url}{path}" response = requests.request(method=method, url=url, data=data, headers=headers, verify=False, **requests_kwargs) # fix encoding of response, based on Accept-Encoding header if "gzip" in headers.get(HEADER_ACCEPT_ENCODING, "").lower(): response._content = gzip.compress(to_bytes(response._content)) response.headers["Content-Length"] = str(len( response._content)) response.headers["Content-Encoding"] = "gzip" return response
def convert_to_binary_event_payload(result, event_type=None, message_type=None): # TODO This encoding has been migrated to the ASF Serializer. # Remove this function after S3 and Kinesis have been migrated to ASF. # e.g.: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html # e.g.: https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html header_descriptors = { ":event-type": event_type or "Records", ":message-type": message_type or "event", } # construct headers headers = b"" for key, value in header_descriptors.items(): header_name = key.encode(DEFAULT_ENCODING) header_value = to_bytes(value) headers += pack("!B", len(header_name)) headers += header_name headers += pack("!B", AWS_BINARY_DATA_TYPE_STRING) headers += pack("!H", len(header_value)) headers += header_value # construct body if isinstance(result, str): body = bytes(result, DEFAULT_ENCODING) else: body = result # calculate lengths headers_length = len(headers) body_length = len(body) # construct message result = pack("!I", body_length + headers_length + 16) result += pack("!I", headers_length) prelude_crc = binascii.crc32(result) result += pack("!I", prelude_crc) result += headers result += body payload_crc = binascii.crc32(result) result += pack("!I", payload_crc) return result
def dispatch_to_backend( context: RequestContext, http_request_dispatcher: Callable[[RequestContext], HttpBackendResponse], include_response_metadata=False, ) -> ServiceResponse: """ Dispatch the given request to a backend by using the `request_forwarder` function to fetch an HTTP response, converting it to a ServiceResponse. :param context: the request context :param http_request_dispatcher: dispatcher that performs the request and returns an HTTP response :param include_response_metadata: whether to include boto3 response metadata in the response :return: """ status, headers, content = http_request_dispatcher(context) operation_model = context.operation response_dict = { # this is what botocore.endpoint.convert_to_response_dict normally does "headers": dict(headers.items()), # boto doesn't like werkzeug headers "status_code": status, "body": to_bytes(content), "context": { "operation_name": operation_model.name, }, } parser = create_parser(context.service.protocol) response = parser.parse(response_dict, operation_model.output_shape) if status >= 301: error = response["Error"] raise CommonServiceException( code=error.get("Code", "UnknownError"), status_code=status, message=error.get("Message", ""), sender_fault=("Type" in error), ) if not include_response_metadata: response.pop("ResponseMetadata", None) return response
def convert_to_binary_event_payload(result, event_type=None, message_type=None): # e.g.: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html # e.g.: https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html header_descriptors = { ":event-type": event_type or "Records", ":message-type": message_type or "event", } # construct headers headers = b"" for key, value in header_descriptors.items(): header_name = key.encode(DEFAULT_ENCODING) header_value = to_bytes(value) headers += pack("!B", len(header_name)) headers += header_name headers += pack("!B", AWS_BINARY_DATA_TYPE_STRING) headers += pack("!H", len(header_value)) headers += header_value # construct body body = bytes(result, DEFAULT_ENCODING) # calculate lengths headers_length = len(headers) body_length = len(body) # construct message result = pack("!I", body_length + headers_length + 16) result += pack("!I", headers_length) prelude_crc = binascii.crc32(result) result += pack("!I", prelude_crc) result += headers result += body payload_crc = binascii.crc32(result) result += pack("!I", payload_crc) return result
def calculate_crc32(content): return crc32(to_bytes(content)) & 0xFFFFFFFF
def send_event_to_target( target_arn: str, event: Dict, target_attributes: Dict = None, asynchronous: bool = True, target: Dict = {}, ): region = target_arn.split(":")[3] if ":lambda:" in target_arn: from localstack.services.awslambda import lambda_api lambda_api.run_lambda(func_arn=target_arn, event=event, context={}, asynchronous=asynchronous) elif ":sns:" in target_arn: sns_client = connect_to_service("sns", region_name=region) sns_client.publish(TopicArn=target_arn, Message=json.dumps(event)) elif ":sqs:" in target_arn: sqs_client = connect_to_service("sqs", region_name=region) queue_url = get_sqs_queue_url(target_arn) msg_group_id = dict_utils.get_safe(target_attributes, "$.SqsParameters.MessageGroupId") kwargs = {"MessageGroupId": msg_group_id} if msg_group_id else {} sqs_client.send_message(QueueUrl=queue_url, MessageBody=json.dumps(event), **kwargs) elif ":states:" in target_arn: stepfunctions_client = connect_to_service("stepfunctions", region_name=region) stepfunctions_client.start_execution(stateMachineArn=target_arn, input=json.dumps(event)) elif ":firehose:" in target_arn: delivery_stream_name = firehose_name(target_arn) firehose_client = connect_to_service("firehose", region_name=region) firehose_client.put_record( DeliveryStreamName=delivery_stream_name, Record={"Data": to_bytes(json.dumps(event))}, ) elif ":events:" in target_arn: if ":api-destination/" in target_arn or ":destination/" in target_arn: send_event_to_api_destination(target_arn, event, target.get("HttpParameters")) else: events_client = connect_to_service("events", region_name=region) eventbus_name = target_arn.split(":")[-1].split("/")[-1] events_client.put_events( Entries=[{ "EventBusName": eventbus_name, "Source": event.get("source"), "DetailType": event.get("detail-type"), "Detail": event.get("detail"), }]) elif ":kinesis:" in target_arn: partition_key_path = dict_utils.get_safe( target_attributes, "$.KinesisParameters.PartitionKeyPath", default_value="$.id", ) stream_name = target_arn.split("/")[-1] partition_key = dict_utils.get_safe(event, partition_key_path, event["id"]) kinesis_client = connect_to_service("kinesis", region_name=region) kinesis_client.put_record( StreamName=stream_name, Data=to_bytes(json.dumps(event)), PartitionKey=partition_key, ) elif ":logs:" in target_arn: log_group_name = target_arn.split(":")[-1] logs_client = connect_to_service("logs", region_name=region) log_stream_name = str(uuid.uuid4()) logs_client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name) logs_client.put_log_events( logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=[{ "timestamp": now_utc(millis=True), "message": json.dumps(event) }], ) else: LOG.warning('Unsupported Events rule target ARN: "%s"', target_arn)
def dummy_wsgi_environment( method: str = "GET", path: str = "", headers: Optional[Union[Dict, Headers]] = None, body: Optional[Union[bytes, str]] = None, scheme: str = "http", root_path: str = "/", query_string: Optional[str] = None, remote_addr: Optional[str] = None, server: Optional[Tuple[str, Optional[int]]] = None, raw_uri: Optional[str] = None, ) -> "WSGIEnvironment": """ Creates a dummy WSGIEnvironment that represents a standalone sans-IO HTTP requests. See https://wsgi.readthedocs.io/en/latest/definitions.html#standard-environ-keys :param method: The HTTP request method (such as GET or POST) :param path: The remainder of the request URL's path. This may be an empty string, if the request URL targets the application root and does not have a trailing slash. :param headers: optional HTTP headers :param body: the body of the request :param scheme: the scheme (http or https) :param root_path: The initial portion of the request URL's path that corresponds to the application object. :param query_string: The portion of the request URL that follows the “?”, if any. May be empty or absent. :param remote_addr: The address making the request :param server: The server (tuple of server name and port) :param raw_uri: The original path that may contain url encoded path elements. :return: A WSGIEnvironment dictionary """ # Standard environ keys environ = { "REQUEST_METHOD": method, # prepare the paths for the "WSGI decoding dance" done by werkzeug "SCRIPT_NAME": unquote(quote(root_path.rstrip("/")), "latin-1"), "PATH_INFO": unquote(quote(path), "latin-1"), "SERVER_PROTOCOL": "HTTP/1.1", } data = strings.to_bytes(body) if body else b"" if query_string is not None: environ["QUERY_STRING"] = query_string if raw_uri: if query_string: raw_uri += "?" + query_string environ["RAW_URI"] = raw_uri environ["REQUEST_URI"] = environ["RAW_URI"] if server: environ["SERVER_NAME"] = server[0] if server[1]: environ["SERVER_PORT"] = str(server[1]) else: environ["SERVER_PORT"] = "80" else: environ["SERVER_NAME"] = "127.0.0.1" environ["SERVER_PORT"] = "80" if remote_addr: environ["REMOTE_ADDR"] = remote_addr if headers: for k, v in headers.items(): name = k.upper().replace("-", "_") if name not in ("CONTENT_TYPE", "CONTENT_LENGTH"): name = f"HTTP_{name}" val = v if name in environ: val = environ[name] + "," + val environ[name] = val if "CONTENT_LENGTH" not in environ: # try to determine content length from body environ["CONTENT_LENGTH"] = str(len(data)) # WSGI environ keys environ["wsgi.version"] = (1, 0) environ["wsgi.url_scheme"] = scheme environ["wsgi.input"] = BytesIO(data) environ["wsgi.input_terminated"] = True environ["wsgi.errors"] = BytesIO() environ["wsgi.multithread"] = True environ["wsgi.multiprocess"] = False environ["wsgi.run_once"] = False return environ
def invoke(fn, self, *args, **kwargs): payload = to_bytes(args[0]) client = aws_stack.connect_to_service("lambda") return client.invoke(FunctionName=self.function_name, Payload=payload)
def invoke(self, invocation_context: ApiInvocationContext): uri = (invocation_context.integration.get("uri") or invocation_context.integration.get("integrationUri") or "") relative_path, query_string_params = extract_query_string_params( path=invocation_context.path_with_query_string) api_id = invocation_context.api_id stage = invocation_context.stage headers = invocation_context.headers resource_path = invocation_context.resource_path invocation_context.context = get_event_request_context( invocation_context) try: path_params = extract_path_params(path=relative_path, extracted_path=resource_path) invocation_context.path_params = path_params except Exception: path_params = {} func_arn = uri if ":lambda:path" in uri: func_arn = uri.split(":lambda:path")[1].split( "functions/")[1].split("/invocations")[0] if invocation_context.authorizer_type: authorizer_context = { invocation_context.authorizer_type: invocation_context.auth_context } invocation_context.context["authorizer"] = authorizer_context payload = self.request_templates.render(invocation_context) # TODO: change this signature to InvocationContext as well! result = lambda_api.process_apigateway_invocation( func_arn, relative_path, payload, stage, api_id, headers, is_base64_encoded=invocation_context.is_data_base64_encoded, path_params=path_params, query_string_params=query_string_params, method=invocation_context.method, resource_path=resource_path, request_context=invocation_context.context, stage_variables=invocation_context.stage_variables, ) if isinstance(result, FlaskResponse): response = flask_to_requests_response(result) elif isinstance(result, Response): response = result else: response = LambdaResponse() parsed_result = result if isinstance(result, dict) else json.loads( str(result or "{}")) parsed_result = common.json_safe(parsed_result) parsed_result = {} if parsed_result is None else parsed_result response.status_code = int(parsed_result.get("statusCode", 200)) parsed_headers = parsed_result.get("headers", {}) if parsed_headers is not None: response.headers.update(parsed_headers) try: result_body = parsed_result.get("body") if isinstance(result_body, dict): response._content = json.dumps(result_body) else: body_bytes = to_bytes(to_str(result_body or "")) if parsed_result.get("isBase64Encoded", False): body_bytes = base64.b64decode(body_bytes) response._content = body_bytes except Exception as e: LOG.warning("Couldn't set Lambda response content: %s", e) response._content = "{}" response.multi_value_headers = parsed_result.get( "multiValueHeaders") or {} # apply custom response template self.update_content_length(response) invocation_context.response = response self.response_templates.render(invocation_context) return invocation_context.response
def start_container( self, container_name_or_id: str, stdin=None, interactive: bool = False, attach: bool = False, flags: Optional[str] = None, ) -> Tuple[bytes, bytes]: LOG.debug("Starting container %s", container_name_or_id) try: container = self.client().containers.get(container_name_or_id) stdout = to_bytes(container_name_or_id) stderr = b"" if interactive or attach: params = {"stdout": 1, "stderr": 1, "stream": 1} if interactive: params["stdin"] = 1 sock = container.attach_socket(params=params) sock = sock._sock if hasattr(sock, "_sock") else sock result_queue = queue.Queue() thread_started = threading.Event() start_waiting = threading.Event() # Note: We need to be careful about potential race conditions here - .wait() should happen right # after .start(). Hence starting a thread and asynchronously waiting for the container exit code def wait_for_result(*_): _exit_code = -1 try: thread_started.set() start_waiting.wait() _exit_code = container.wait()["StatusCode"] except APIError as e: _exit_code = 1 raise ContainerException(str(e)) finally: result_queue.put(_exit_code) # start listener thread start_worker_thread(wait_for_result) thread_started.wait() # start container container.start() # start awaiting container result start_waiting.set() # handle container input/output # under windows, the socket has no __enter__ / cannot be used as context manager # therefore try/finally instead of with here try: if stdin: sock.sendall(to_bytes(stdin)) sock.shutdown(socket.SHUT_WR) stdout, stderr = self._read_from_sock(sock, False) except socket.timeout: LOG.debug( f"Socket timeout when talking to the I/O streams of Docker container '{container_name_or_id}'" ) finally: sock.close() # get container exit code exit_code = result_queue.get() if exit_code: raise ContainerException( f"Docker container returned with exit code {exit_code}", stdout=stdout, stderr=stderr, ) else: container.start() return stdout, stderr except NotFound: raise NoSuchContainer(container_name_or_id) except APIError as e: raise ContainerException() from e
def roundtrip(data): encoded = base64.urlsafe_b64encode(to_bytes(data)) result = base64_decode(encoded) assert to_bytes(data) == result
def _in_path_or_payload(search_str): return to_str(search_str) in path or to_bytes(search_str) in data_bytes
def get_api_from_custom_rules(method, path, data, headers): """Determine backend port based on custom rules.""" # API Gateway invocation URLs if ("/%s/" % PATH_USER_REQUEST) in path: return "apigateway", config.service_port("apigateway") # detect S3 presigned URLs if "AWSAccessKeyId=" in path or "Signature=" in path: return "s3", config.service_port("s3") # heuristic for SQS queue URLs if is_sqs_queue_url(path): return "sqs", config.service_port("sqs") # DynamoDB shell URLs if path.startswith("/shell") or path.startswith("/dynamodb/shell"): return "dynamodb", config.service_port("dynamodb") data_bytes = to_bytes(data or "") version, action = extract_version_and_action(path, data_bytes) def _in_path_or_payload(search_str): return to_str(search_str) in path or to_bytes(search_str) in data_bytes if path == "/" and b"QueueName=" in data_bytes: return "sqs", config.service_port("sqs") if "Action=ConfirmSubscription" in path: return "sns", config.service_port("sns") if path.startswith("/2015-03-31/functions/"): return "lambda", config.service_port("lambda") if _in_path_or_payload("Action=AssumeRoleWithWebIdentity"): return "sts", config.service_port("sts") if _in_path_or_payload("Action=AssumeRoleWithSAML"): return "sts", config.service_port("sts") if _in_path_or_payload("Action=AssumeRole"): return "sts", config.service_port("sts") # SQS queue requests if _in_path_or_payload("QueueUrl=") and _in_path_or_payload("Action="): return "sqs", config.service_port("sqs") if matches_service_action("sqs", action, version=version): return "sqs", config.service_port("sqs") # SNS topic requests if matches_service_action("sns", action, version=version): return "sns", config.service_port("sns") # TODO: move S3 public URLs to a separate port/endpoint, OR check ACLs here first stripped = path.strip("/") if method in ["GET", "HEAD"] and stripped: # assume that this is an S3 GET request with URL path `/<bucket>/<key ...>` return "s3", config.service_port("s3") # detect S3 URLs if stripped and "/" not in stripped: if method == "PUT": # assume that this is an S3 PUT bucket request with URL path `/<bucket>` return "s3", config.service_port("s3") if method == "POST" and is_s3_form_data(data_bytes): # assume that this is an S3 POST request with form parameters or multipart form in the body return "s3", config.service_port("s3") # detect S3 requests sent from aws-cli using --no-sign-request option if "aws-cli/" in headers.get("User-Agent", ""): return "s3", config.service_port("s3") # S3 delete object requests if ( method == "POST" and "delete=" in path and b"<Delete" in data_bytes and b"<Key>" in data_bytes ): return "s3", config.service_port("s3") # Put Object API can have multiple keys if stripped.count("/") >= 1 and method == "PUT": # assume that this is an S3 PUT bucket object request with URL path `/<bucket>/object` # or `/<bucket>/object/object1/+` return "s3", config.service_port("s3") auth_header = headers.get("Authorization") or "" # detect S3 requests with "AWS id:key" Auth headers if auth_header.startswith("AWS "): return "s3", config.service_port("s3") # certain EC2 requests from Java SDK contain no Auth headers (issue #3805) if b"Version=2016-11-15" in data_bytes: return "ec2", config.service_port("ec2")
def get_hash(value) -> str: max_length = 10 digest = hashlib.sha1() digest.update(to_bytes(str(value))) result = digest.hexdigest() return result[:max_length]
def legacy_rules(request: Request) -> Optional[str]: """ *Legacy* rules which migrate routing logic which will become obsolete with the ASF Gateway. All rules which are implemented here should be migrated to the new router once these services are migrated to ASF. TODO: These custom rules should become obsolete by migrating these to use the http/router.py """ path = request.path method = request.method host = hostname_from_url(request.host) # API Gateway invocation URLs # TODO: deprecated with #6040, where API GW user routes are served through the gateway directly if ("/%s/" % PATH_USER_REQUEST) in request.path or ( host.endswith(LOCALHOST_HOSTNAME) and "execute-api" in host): return "apigateway" # DynamoDB shell URLs if path.startswith("/shell") or path.startswith("/dynamodb/shell"): return "dynamodb" # TODO The remaining rules here are special S3 rules - needs to be discussed how these should be handled. # Some are similar to other rules and not that greedy, others are nearly general fallbacks. stripped = path.strip("/") if method in ["GET", "HEAD"] and stripped: # assume that this is an S3 GET request with URL path `/<bucket>/<key ...>` return "s3" # detect S3 URLs if stripped and "/" not in stripped: if method == "PUT": # assume that this is an S3 PUT bucket request with URL path `/<bucket>` return "s3" if method == "POST" and "key" in request.values: # assume that this is an S3 POST request with form parameters or multipart form in the body return "s3" # detect S3 requests sent from aws-cli using --no-sign-request option if "aws-cli/" in str(request.user_agent): return "s3" # detect S3 pre-signed URLs (v2 and v4) values = request.values if any(value in values for value in [ "AWSAccessKeyId", "Signature", "X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date", "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Signature", ]): return "s3" # S3 delete object requests if method == "POST" and "delete" in values: data_bytes = to_bytes(request.data) if b"<Delete" in data_bytes and b"<Key>" in data_bytes: return "s3" # Put Object API can have multiple keys if stripped.count("/") >= 1 and method == "PUT": # assume that this is an S3 PUT bucket object request with URL path `/<bucket>/object` # or `/<bucket>/object/object1/+` return "s3" # detect S3 requests with "AWS id:key" Auth headers auth_header = request.headers.get("Authorization") or "" if auth_header.startswith("AWS "): return "s3" if uses_host_addressing(request.headers): # Note: This needs to be the last rule (and therefore is not in the host rules), since it is incredibly greedy return "s3"
# maps service names/versions to list of action names SERVICE_ACTIONS_CACHE: Dict[str, Set[str]] = {} # default service versions DEFAULT_SERVICE_VERSIONS: Dict[str, str] = { "sns": "2010-03-31", "sqs": "2012-11-05", } # regexes to extract info from URL paths / payloads _REGEX_ACTION = r"(^|.*\?|.*&)Action=([a-zA-Z0-9_]+)($|&)" _REGEX_VERSION = r"(^|.*\?|.*&)Version=([a-zA-Z0-9_]+)($|&)" REGEX_ACTION: re.Pattern = re.compile(_REGEX_ACTION) REGEX_VERSION: re.Pattern = re.compile(_REGEX_VERSION) REGEXB_ACTION: re.Pattern = re.compile(to_bytes(_REGEX_ACTION)) REGEXB_VERSION: re.Pattern = re.compile(to_bytes(_REGEX_VERSION)) # TODO: Add more comprehensive tests for AWS SDK v2. It seems that # the v2 SDK (e.g., from Java) in certain configurations is not sending # the Authorization header we depend on. We can use some of the heuristics # below for routing requests to the correct target services (based on # 'Action' or 'Version' attributes in the request), but the more severe issue # seems to be that the region info is not being transmitted in certain # situations. If this turns out to be true, then we may need to think about # a more comprehensive refactoring of our routing/region-targeting approach. def get_service_action_names(service: str, version: str = None) -> Set[str]: """Returns, for a given service name and version, the list of available service action names.""" version = version or DEFAULT_SERVICE_VERSIONS.get(service)
def test_kinesis_event_source_mapping_with_on_failure_destination_config( self, lambda_client, create_lambda_function, sqs_client, sqs_queue_arn, sqs_create_queue, create_iam_role_with_policy, kinesis_client, wait_for_stream_ready, ): try: function_name = f"lambda_func-{short_uid()}" role = f"test-lambda-role-{short_uid()}" policy_name = f"test-lambda-policy-{short_uid()}" kinesis_name = f"test-kinesis-{short_uid()}" role_arn = create_iam_role_with_policy( RoleName=role, PolicyName=policy_name, RoleDefinition=lambda_role, PolicyDefinition=s3_lambda_permission, ) create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON37, role=role_arn, ) kinesis_client.create_stream(StreamName=kinesis_name, ShardCount=1) result = kinesis_client.describe_stream( StreamName=kinesis_name)["StreamDescription"] kinesis_arn = result["StreamARN"] wait_for_stream_ready(stream_name=kinesis_name) queue_event_source_mapping = sqs_create_queue() destination_queue = sqs_queue_arn(queue_event_source_mapping) destination_config = { "OnFailure": { "Destination": destination_queue } } message = { "input": "hello", "value": "world", lambda_integration.MSG_BODY_RAISE_ERROR_FLAG: 1, } result = lambda_client.create_event_source_mapping( FunctionName=function_name, BatchSize=1, StartingPosition="LATEST", EventSourceArn=kinesis_arn, MaximumBatchingWindowInSeconds=1, MaximumRetryAttempts=1, DestinationConfig=destination_config, ) event_source_mapping_uuid = result["UUID"] _await_event_source_mapping_enabled(lambda_client, event_source_mapping_uuid) kinesis_client.put_record(StreamName=kinesis_name, Data=to_bytes(json.dumps(message)), PartitionKey="custom") def verify_failure_received(): result = sqs_client.receive_message( QueueUrl=queue_event_source_mapping) msg = result["Messages"][0] body = json.loads(msg["Body"]) assert body["requestContext"][ "condition"] == "RetryAttemptsExhausted" assert body["KinesisBatchInfo"]["batchSize"] == 1 assert body["KinesisBatchInfo"]["streamArn"] == kinesis_arn retry(verify_failure_received, retries=50, sleep=5, sleep_before=5) finally: kinesis_client.delete_stream(StreamName=kinesis_name, EnforceConsumerDeletion=True) lambda_client.delete_event_source_mapping( UUID=event_source_mapping_uuid)
def calculate_crc32(response): return crc32(to_bytes(response.content)) & 0xFFFFFFFF