def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = json.loads(to_str(data))
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM
if action == ACTION_CREATE_STREAM else
event_publisher.EVENT_KINESIS_DELETE_STREAM)
payload = {'n': event_publisher.get_hash(data.get('StreamName'))}
if action == ACTION_CREATE_STREAM:
payload['s'] = data.get('ShardCount')
event_publisher.fire_event(event_type, payload=payload)
elif action == ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber':
response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_UPDATE_SHARD_COUNT:
# Currently kinesalite, which backs the Kinesis implementation for localstack, does
# not support UpdateShardCount:
# https://github.com/mhart/kinesalite/issues/61
#
# [Terraform](https://www.terraform.io) makes the call to UpdateShardCount when it
# applies Kinesis resources. A Terraform run fails when this is not present.
#
# The code that follows just returns a successful response, bypassing the 400
# response that kinesalite returns.
#
response = Response()
response.status_code = 200
content = {
'CurrentShardCount': 1,
'StreamName': data['StreamName'],
'TargetShardCount': data['TargetShardCount']
}
response.encoding = 'UTF-8'
response._content = json.dumps(content)
return response
def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = json.loads(data)
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM if action == ACTION_CREATE_STREAM
else event_publisher.EVENT_KINESIS_DELETE_STREAM)
event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(data.get('StreamName'))})
elif action == ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber': response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = json.loads(to_str(data))
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM if action == ACTION_CREATE_STREAM
else event_publisher.EVENT_KINESIS_DELETE_STREAM)
event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(data.get('StreamName'))})
elif action == ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber': response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
def update_kinesis(method,
path,
data,
headers,
response=None,
return_forward_info=False):
action = headers['X-Amz-Target'] if 'X-Amz-Target' in headers else None
if return_forward_info:
if random.random() < config.KINESIS_ERROR_PROBABILITY:
return kinesis_error_response(data)
return True
records = []
if action == constants.KINESIS_ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == constants.KINESIS_ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber': response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target', '').split('.')[-1]
data, encoding_type = self.decode_content(data or '{}', True)
response._content = self.replace_in_encoded(response.content or '')
records = []
if action in ('CreateStream', 'DeleteStream'):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM
if action == 'CreateStream' else
event_publisher.EVENT_KINESIS_DELETE_STREAM)
payload = {'n': event_publisher.get_hash(data.get('StreamName'))}
if action == 'CreateStream':
payload['s'] = data.get('ShardCount')
event_publisher.fire_event(event_type, payload=payload)
elif action == 'PutRecord':
response_body = self.decode_content(response.content)
# Note: avoid adding 'encryptionType':'NONE' in the event_record, as this breaks .NET Lambdas
event_record = {
'approximateArrivalTimestamp': epoch_timestamp(),
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == 'PutRecords':
event_records = []
response_body = self.decode_content(response.content)
if 'Records' in response_body:
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
# Note: avoid adding 'encryptionType':'NONE' in the event_record, as this breaks .NET Lambdas
event_record = {
'approximateArrivalTimestamp':
epoch_timestamp(),
'data':
record['Data'],
'partitionKey':
record['PartitionKey'],
'sequenceNumber':
response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == 'UpdateShardCount':
# Currently kinesalite, which backs the Kinesis implementation for localstack, does
# not support UpdateShardCount:
# https://github.com/mhart/kinesalite/issues/61
#
# [Terraform](https://www.terraform.io) makes the call to UpdateShardCount when it
# applies Kinesis resources. A Terraform run fails when this is not present.
#
# The code that follows just returns a successful response, bypassing the 400
# response that kinesalite returns.
#
response = Response()
response.status_code = 200
content = {
'CurrentShardCount': 1,
'StreamName': data['StreamName'],
'TargetShardCount': data['TargetShardCount']
}
response.encoding = 'UTF-8'
response._content = json.dumps(content)
return response
elif action == 'GetRecords':
sdk_v2 = self.sdk_is_v2(
headers.get('User-Agent', '').split(' ')[0])
results, encoding_type = self.decode_content(
response.content, True)
records = results.get('Records', [])
if not records:
return response
for record in records:
if sdk_v2:
record['ApproximateArrivalTimestamp'] = int(
record['ApproximateArrivalTimestamp'])
if not isinstance(record['Data'], str):
# Remove double quotes from data written as bytes
# https://github.com/localstack/localstack/issues/3588
tmp = bytearray(record['Data']['data'])
if len(tmp) >= 2 and tmp[0] == tmp[-1] == b'"'[0]:
tmp = tmp[1:-1]
if encoding_type == APPLICATION_JSON:
record['Data'] = to_str(base64.b64encode(tmp))
else:
record['Data'] = to_str(tmp)
else:
tmp = base64.b64decode(record['Data'])
if len(tmp) >= 2 and tmp[0] == tmp[-1] == b'"'[0]:
tmp = tmp[1:-1]
record['Data'] = to_str(base64.b64encode(tmp))
response._content = cbor2.dumps(
results) if encoding_type == APPLICATION_CBOR else json.dumps(
results)
return response
def return_response(self, method, path, data, headers, response):
action = headers.get("X-Amz-Target", "").split(".")[-1]
data, encoding_type = self.decode_content(data or "{}", True)
response._content = self.replace_in_encoded(response.content or "")
if action in ("CreateStream", "DeleteStream"):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM
if action == "CreateStream" else
event_publisher.EVENT_KINESIS_DELETE_STREAM)
payload = {"n": event_publisher.get_hash(data.get("StreamName"))}
if action == "CreateStream":
payload["s"] = data.get("ShardCount")
event_publisher.fire_event(event_type, payload=payload)
elif action == "PutRecord":
response_body = self.decode_content(response.content)
# Note: avoid adding 'encryptionType':'NONE' in the event_record, as this breaks .NET Lambdas
event_record = {
"approximateArrivalTimestamp": epoch_timestamp(),
"data": data["Data"],
"partitionKey": data["PartitionKey"],
"sequenceNumber": response_body.get("SequenceNumber"),
}
event_records = [event_record]
stream_name = data["StreamName"]
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == "PutRecords":
event_records = []
response_body = self.decode_content(response.content)
if "Records" in response_body:
response_records = response_body["Records"]
records = data["Records"]
for index in range(0, len(records)):
record = records[index]
# Note: avoid adding 'encryptionType':'NONE' in the event_record, as this breaks .NET Lambdas
event_record = {
"approximateArrivalTimestamp":
epoch_timestamp(),
"data":
record["Data"],
"partitionKey":
record["PartitionKey"],
"sequenceNumber":
response_records[index].get("SequenceNumber"),
}
event_records.append(event_record)
stream_name = data["StreamName"]
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == "UpdateShardCount" and config.KINESIS_PROVIDER == "kinesalite":
# Currently kinesalite, which backs the Kinesis implementation for localstack, does
# not support UpdateShardCount:
# https://github.com/mhart/kinesalite/issues/61
#
# [Terraform](https://www.terraform.io) makes the call to UpdateShardCount when it
# applies Kinesis resources. A Terraform run fails when this is not present.
#
# The code that follows just returns a successful response, bypassing the 400
# response that kinesalite returns.
#
response = Response()
response.status_code = 200
content = {
"CurrentShardCount": 1,
"StreamName": data["StreamName"],
"TargetShardCount": data["TargetShardCount"],
}
response.encoding = "UTF-8"
response._content = json.dumps(content)
return response
elif action == "GetRecords":
sdk_v2 = self.is_sdk_v2_request(headers)
results, encoding_type = self.decode_content(
response.content, True)
records = results.get("Records", [])
if not records:
return response
for record in records:
if sdk_v2:
record["ApproximateArrivalTimestamp"] = int(
record["ApproximateArrivalTimestamp"])
tmp = record["Data"]
# "tmp" is either a base64-encoded string, or a dict {"data": <data_bytes...>}
is_base64_encoded = isinstance(tmp, str)
if isinstance(tmp, dict):
tmp = bytearray(tmp.get("data", []))
if is_base64_encoded:
tmp = base64.b64decode(tmp)
if encoding_type == APPLICATION_JSON:
# Remove double quotes from data written in regular JSON encoding (not for CBOR!)
# https://github.com/localstack/localstack/issues/3588
if len(tmp) >= 2 and tmp[0] == tmp[-1] == b'"'[0]:
tmp = tmp[1:-1]
if encoding_type == APPLICATION_CBOR:
# Note: AWS Java SDK requires bytes embedded in CBOR encoding, not base64 strings!
tmp = to_bytes(tmp)
if encoding_type == APPLICATION_JSON:
if is_base64_encoded:
tmp = base64.b64encode(tmp)
tmp = to_str(tmp)
record["Data"] = tmp
response._content = encode_data(results, encoding_type)
return response
if response.status_code >= 400:
response_body = self.decode_content(response.content)
if (response_body and response_body.get("__type")
and not headers.get(HEADER_AMZN_ERROR_TYPE)):
response.headers[HEADER_AMZN_ERROR_TYPE] = str(
response_body.get("__type"))
def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = self.decode_content(data or '{}')
response._content = self.replace_in_encoded(response.content or '')
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM if action == ACTION_CREATE_STREAM
else event_publisher.EVENT_KINESIS_DELETE_STREAM)
payload = {'n': event_publisher.get_hash(data.get('StreamName'))}
if action == ACTION_CREATE_STREAM:
payload['s'] = data.get('ShardCount')
event_publisher.fire_event(event_type, payload=payload)
elif action == ACTION_PUT_RECORD:
response_body = self.decode_content(response.content)
# Note: avoid adding 'encryptionType':'NONE' in the event_record, as this breaks .NET Lambdas
event_record = {
'approximateArrivalTimestamp': epoch_timestamp(),
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = self.decode_content(response.content)
if 'Records' in response_body:
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
# Note: avoid adding 'encryptionType':'NONE' in the event_record, as this breaks .NET Lambdas
event_record = {
'approximateArrivalTimestamp': epoch_timestamp(),
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber': response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_UPDATE_SHARD_COUNT:
# Currently kinesalite, which backs the Kinesis implementation for localstack, does
# not support UpdateShardCount:
# https://github.com/mhart/kinesalite/issues/61
#
# [Terraform](https://www.terraform.io) makes the call to UpdateShardCount when it
# applies Kinesis resources. A Terraform run fails when this is not present.
#
# The code that follows just returns a successful response, bypassing the 400
# response that kinesalite returns.
#
response = Response()
response.status_code = 200
content = {
'CurrentShardCount': 1,
'StreamName': data['StreamName'],
'TargetShardCount': data['TargetShardCount']
}
response.encoding = 'UTF-8'
response._content = json.dumps(content)
return response
elif action == ACTION_GET_RECORDS:
sdk_v2 = self.sdk_is_v2(headers.get('User-Agent', '').split(' ')[0])
results, encoding_type = self.decode_content(response.content, True)
for record in results['Records']:
if sdk_v2:
record['ApproximateArrivalTimestamp'] = int(record['ApproximateArrivalTimestamp'] * 1000)
if not isinstance(record['Data'], str):
record['Data'] = base64.encodebytes(bytearray(record['Data']['data']))
if encoding_type == APPLICATION_CBOR:
response._content = cbor2.dumps(results)
else:
response._content = json.dumps(results)
return response
