def test_events_written_to_disk_are_timestamp_prefixed_for_chronological_ordering(
self):
event_type = str(uuid.uuid4())
event_details_to_publish = list(
map(lambda n: 'event %s' % n, range(10)))
for detail in event_details_to_publish:
self.events_client.put_events(Entries=[{
'Source': 'unittest',
'Resources': [],
'DetailType': event_type,
'Detail': json_safe(detail)
}])
sorted_events_written_to_disk = map(
lambda filename: json.loads(
str(load_file(os.path.join(EVENTS_TMP_DIR, filename)))),
sorted(os.listdir(EVENTS_TMP_DIR)))
sorted_events = list(
filter(lambda event: event['DetailType'] == event_type,
sorted_events_written_to_disk))
self.assertListEqual(
event_details_to_publish,
list(map(lambda event: json_safe(event['Detail']), sorted_events)))
def test_non_ascii_chars(self):
aws_stack.create_dynamodb_table(TEST_DDB_TABLE_NAME,
partition_key=PARTITION_KEY)
table = self.dynamodb.Table(TEST_DDB_TABLE_NAME)
# write some items containing non-ASCII characters
items = {
'id1': {
PARTITION_KEY: 'id1',
'data': 'foobar123 ✓'
},
'id2': {
PARTITION_KEY: 'id2',
'data': 'foobar123 £'
},
'id3': {
PARTITION_KEY: 'id3',
'data': 'foobar123 ¢'
}
}
for k, item in items.items():
table.put_item(Item=item)
for item_id in items.keys():
item = table.get_item(Key={PARTITION_KEY: item_id})['Item']
# need to fix up the JSON and convert str to unicode for Python 2
item1 = json_safe(item)
item2 = json_safe(items[item_id])
self.assertEqual(item1, item2)
# clean up
delete_table(TEST_DDB_TABLE_NAME)
def test_non_ascii_chars(self, dynamodb):
aws_stack.create_dynamodb_table(TEST_DDB_TABLE_NAME,
partition_key=PARTITION_KEY)
table = dynamodb.Table(TEST_DDB_TABLE_NAME)
# write some items containing non-ASCII characters
items = {
"id1": {
PARTITION_KEY: "id1",
"data": "foobar123 ✓"
},
"id2": {
PARTITION_KEY: "id2",
"data": "foobar123 £"
},
"id3": {
PARTITION_KEY: "id3",
"data": "foobar123 ¢"
},
}
for k, item in items.items():
table.put_item(Item=item)
for item_id in items.keys():
item = table.get_item(Key={PARTITION_KEY: item_id})["Item"]
# need to fix up the JSON and convert str to unicode for Python 2
item1 = json_safe(item)
item2 = json_safe(items[item_id])
assert item1 == item2
# clean up
delete_table(TEST_DDB_TABLE_NAME)
def execute_java_lambda(self,
event,
context,
main_file,
func_details=None):
handler = func_details.handler
opts = config.LAMBDA_JAVA_OPTS if config.LAMBDA_JAVA_OPTS else ""
event_file = EVENT_FILE_PATTERN.replace("*", short_uid())
save_file(event_file, json.dumps(json_safe(event)))
TMP_FILES.append(event_file)
class_name = handler.split("::")[0]
classpath = "%s:%s:%s" % (
main_file,
Util.get_java_classpath(main_file),
LAMBDA_EXECUTOR_JAR,
)
cmd = "java %s -cp %s %s %s %s" % (
opts,
classpath,
LAMBDA_EXECUTOR_CLASS,
class_name,
event_file,
)
LOG.info(cmd)
result = self.run_lambda_executor(cmd, func_details=func_details)
return result
def store_delivery_log(
subscriber: dict, success: bool, message: str, message_id: str, delivery: dict = None
):
log_group_name = subscriber.get("TopicArn", "").replace("arn:aws:", "").replace(":", "/")
log_stream_name = long_uid()
invocation_time = int(time.time() * 1000)
delivery = not_none_or(delivery, {})
delivery["deliveryId"] = (long_uid(),)
delivery["destination"] = (subscriber.get("Endpoint", ""),)
delivery["dwellTimeMs"] = 200
if not success:
delivery["attemps"] = 1
delivery_log = {
"notification": {
"messageMD5Sum": md5(message),
"messageId": message_id,
"topicArn": subscriber.get("TopicArn"),
"timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f%z"),
},
"delivery": delivery,
"status": "SUCCESS" if success else "FAILURE",
}
log_output = json.dumps(json_safe(delivery_log))
return store_cloudwatch_logs(log_group_name, log_stream_name, log_output, invocation_time)
def test_put_events_with_target_sqs(self):
queue_name = 'queue-{}'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
bus_name = 'bus-{}'.format(short_uid())
sqs_client = aws_stack.connect_to_service('sqs')
queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN))
rs = self.events_client.put_targets(Rule=rule_name,
EventBusName=bus_name,
Targets=[{
'Id': target_id,
'Arn': queue_arn
}])
self.assertIn('FailedEntryCount', rs)
self.assertIn('FailedEntries', rs)
self.assertEqual(rs['FailedEntryCount'], 0)
self.assertEqual(rs['FailedEntries'], [])
self.events_client.put_events(
Entries=[{
'EventBusName': bus_name,
'Source': TEST_EVENT_PATTERN['Source'][0],
'DetailType': TEST_EVENT_PATTERN['detail-type'][0],
'Detail': json_safe(TEST_EVENT_PATTERN['Detail'][0])
}])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp['Messages']
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(len(messages), 1)
actual_event = json.loads(messages[0]['Body'])
self.assertIsValidEvent(actual_event)
self.assertEqual(actual_event['detail'],
TEST_EVENT_PATTERN['Detail'][0])
# clean up
sqs_client.delete_queue(QueueUrl=queue_url)
self.events_client.remove_targets(Rule=rule_name,
EventBusName=bus_name,
Ids=[target_id],
Force=True)
self.events_client.delete_rule(Name=rule_name,
EventBusName=bus_name,
Force=True)
self.events_client.delete_event_bus(Name=bus_name)
def send_events():
yield convert_to_binary_event_payload('', event_type='initial-response')
iter = iterator
last_sequence_number = starting_sequence_number
# TODO: find better way to run loop up to max 5 minutes (until connection terminates)!
for i in range(5 * 60):
result = None
try:
result = kinesis.get_records(ShardIterator=iter)
except Exception as e:
if 'ResourceNotFoundException' in str(e):
LOG.debug('Kinesis stream "%s" has been deleted, closing shard subscriber' % stream_name)
return
raise
iter = result.get('NextShardIterator')
records = result.get('Records', [])
for record in records:
record['ApproximateArrivalTimestamp'] = record['ApproximateArrivalTimestamp'].timestamp()
if data_needs_encoding:
record['Data'] = base64.b64encode(record['Data'])
record['Data'] = to_str(record['Data'])
last_sequence_number = record['SequenceNumber']
if not records:
time.sleep(1)
continue
response = {
'ChildShards': [],
'ContinuationSequenceNumber': last_sequence_number,
'MillisBehindLatest': 0,
'Records': json_safe(records),
}
result = json.dumps(response)
yield convert_to_binary_event_payload(result, event_type='SubscribeToShardEvent')
def _execute(self, func_arn, func_details, event, context=None, version=None):
runtime = func_details.runtime
handler = func_details.handler
environment = self._prepare_environment(func_details)
# configure USE_SSL in environment
if config.USE_SSL:
environment["USE_SSL"] = "1"
# prepare event body
if not event:
LOG.info('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
main_endpoint = get_main_endpoint_from_container()
environment["LOCALSTACK_HOSTNAME"] = main_endpoint
environment["EDGE_PORT"] = str(config.EDGE_PORT)
environment["_HANDLER"] = handler
if os.environ.get("HTTP_PROXY"):
environment["HTTP_PROXY"] = os.environ["HTTP_PROXY"]
if func_details.timeout:
environment["AWS_LAMBDA_FUNCTION_TIMEOUT"] = str(func_details.timeout)
if context:
environment["AWS_LAMBDA_FUNCTION_NAME"] = context.function_name
environment["AWS_LAMBDA_FUNCTION_VERSION"] = context.function_version
environment["AWS_LAMBDA_FUNCTION_INVOKED_ARN"] = context.invoked_function_arn
environment["AWS_LAMBDA_COGNITO_IDENTITY"] = json.dumps(context.cognito_identity or {})
if context.client_context is not None:
environment["AWS_LAMBDA_CLIENT_CONTEXT"] = json.dumps(
to_str(base64.b64decode(to_bytes(context.client_context)))
)
# pass JVM options to the Lambda environment, if configured
if config.LAMBDA_JAVA_OPTS and is_java_lambda(runtime):
if environment.get("JAVA_TOOL_OPTIONS"):
LOG.info(
"Skip setting LAMBDA_JAVA_OPTS as JAVA_TOOL_OPTIONS already defined in Lambda env vars"
)
else:
LOG.debug(
"Passing JVM options to container environment: JAVA_TOOL_OPTIONS=%s"
% config.LAMBDA_JAVA_OPTS
)
environment["JAVA_TOOL_OPTIONS"] = config.LAMBDA_JAVA_OPTS
# accept any self-signed certificates for outgoing calls from the Lambda
if is_nodejs_runtime(runtime):
environment["NODE_TLS_REJECT_UNAUTHORIZED"] = "0"
# run Lambda executor and fetch invocation result
LOG.info("Running lambda: %s" % func_details.arn())
result = self.run_lambda_executor(
event=stdin, env_vars=environment, func_details=func_details
)
return result
def _execute(self, func_arn, func_details, event, context=None, version=None):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = func_details.envvars.copy()
# configure USE_SSL in environment
if config.USE_SSL:
environment['USE_SSL'] = '1'
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
docker_host = config.DOCKER_HOST_FROM_CONTAINER
environment['HOSTNAME'] = docker_host
environment['LOCALSTACK_HOSTNAME'] = docker_host
environment['_HANDLER'] = handler
if func_details.timeout:
environment['AWS_LAMBDA_FUNCTION_TIMEOUT'] = str(func_details.timeout)
if context:
environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name
environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version
environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn
# custom command to execute in the container
command = ''
# if running a Java Lambda, set up classpath arguments
if is_java_lambda(runtime):
java_opts = Util.get_java_opts()
stdin = None
# copy executor jar into temp directory
target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR))
if not os.path.exists(target_file):
cp_r(LAMBDA_EXECUTOR_JAR, target_file)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body)
classpath = Util.get_java_classpath(target_file)
command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" %
(taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE))
# accept any self-signed certificates for outgoing calls from the Lambda
if is_nodejs_runtime(runtime):
environment['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd)
# lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
LOG.info('Running lambda cmd: %s' % cmd)
result = self.run_lambda_executor(cmd, stdin, env_vars=environment, func_details=func_details)
return result
def create_key(data: Dict):
key_usage = data.get("KeyUsage")
if key_usage != "SIGN_VERIFY":
return
data["KeyId"] = short_uid()
result = _generate_data_key_pair(data, create_cipher=False)
result = {"KeyMetadata": json_safe(result)}
return result
def test_convert_yaml_date_strings(self):
yaml_source = 'Version: 2012-10-17'
obj = yaml.safe_load(yaml_source)
self.assertIn(type(obj['Version']), (datetime.date, str))
if isinstance(obj['Version'], datetime.date):
obj = json_safe(obj)
self.assertEqual(str, type(obj['Version']))
self.assertEqual('2012-10-17', obj['Version'])
def forward_request(self, method, path, data, headers):
global STREAM_CONSUMERS
data = self.decode_content(data or '{}')
action = headers.get('X-Amz-Target', '').split('.')[-1]
if action == 'RegisterStreamConsumer':
consumer = clone(data)
consumer['ConsumerStatus'] = 'ACTIVE'
consumer['ConsumerARN'] = '%s/consumer/%s' % (data['StreamARN'],
data['ConsumerName'])
consumer['ConsumerCreationTimestamp'] = float(now_utc())
consumer = json_safe(consumer)
STREAM_CONSUMERS.append(consumer)
return {'Consumer': consumer}
elif action == 'DeregisterStreamConsumer':
def consumer_matches(c):
stream_arn = data.get('StreamARN')
cons_name = data.get('ConsumerName')
cons_arn = data.get('ConsumerARN')
return (c.get('ConsumerARN') == cons_arn
or (c.get('StreamARN') == stream_arn
and c.get('ConsumerName') == cons_name))
STREAM_CONSUMERS = [
c for c in STREAM_CONSUMERS if not consumer_matches(c)
]
return {}
elif action == 'ListStreamConsumers':
result = {
'Consumers': [
c for c in STREAM_CONSUMERS
if c.get('StreamARN') == data.get('StreamARN')
]
}
return result
elif action == 'DescribeStreamConsumer':
consumer_arn = data.get('ConsumerARN') or data['ConsumerName']
consumer_name = data.get('ConsumerName') or data['ConsumerARN']
creation_timestamp = data.get('ConsumerCreationTimestamp')
result = {
'ConsumerDescription': {
'ConsumerARN': consumer_arn,
'ConsumerCreationTimestamp': creation_timestamp,
'ConsumerName': consumer_name,
'ConsumerStatus': 'ACTIVE',
'StreamARN': data.get('StreamARN')
}
}
return result
elif action == 'SubscribeToShard':
result = subscribe_to_shard(data)
return result
if random.random() < config.KINESIS_ERROR_PROBABILITY:
if action in ['PutRecord', 'PutRecords']:
return kinesis_error_response(data, action)
return True
def set_response_content(response, content, headers=None):
if isinstance(content, dict):
content = json.dumps(json_safe(content))
elif isinstance(content, RequestsResponse):
response.status_code = content.status_code
content = content.content
response._content = content or ""
response.headers.update(headers or {})
response.headers["Content-Length"] = str(len(response._content))
def forward_request(self, **kwargs):
response = Response()
response.status_code = 200
result = {
'data': kwargs.get('data') or '{}',
'headers': dict(kwargs.get('headers'))
}
response._content = json.dumps(json_safe(result))
return response
def replace(params, **kwargs):
result = param_func(params, **kwargs)
for name in param_names:
if isinstance(result.get(name), (dict, list)):
# Fix for https://github.com/localstack/localstack/issues/2022
# Convert any date instances to date strings, etc, Version: "2012-10-17"
param_value = common.json_safe(result[name])
result[name] = json.dumps(param_value)
return result
def create_key(data: Dict):
key_usage = data.get("KeyUsage")
if key_usage != "SIGN_VERIFY" or KMS_PROVIDER == "local-kms":
return
data["KeyId"] = long_uid()
result = _generate_data_key_pair(data, create_cipher=False)
result = {"KeyMetadata": json_safe(result)}
return result
def forward_request(self, method, path, data, headers):
global STREAM_CONSUMERS
data = json.loads(to_str(data or '{}'))
action = headers.get('X-Amz-Target')
if action == '%s.RegisterStreamConsumer' % ACTION_PREFIX:
consumer = clone(data)
consumer['ConsumerStatus'] = 'ACTIVE'
consumer['ConsumerARN'] = '%s/consumer/%s' % (data['StreamARN'],
data['ConsumerName'])
consumer['ConsumerCreationTimestamp'] = datetime.now()
consumer = json_safe(consumer)
STREAM_CONSUMERS.append(consumer)
return {'Consumer': consumer}
elif action == '%s.DeregisterStreamConsumer' % ACTION_PREFIX:
def consumer_matches(c):
stream_arn = data.get('StreamARN')
cons_name = data.get('ConsumerName')
cons_arn = data.get('ConsumerARN')
return (c.get('ConsumerARN') == cons_arn
or (c.get('StreamARN') == stream_arn
and c.get('ConsumerName') == cons_name))
STREAM_CONSUMERS = [
c for c in STREAM_CONSUMERS if not consumer_matches(c)
]
return {}
elif action == '%s.ListStreamConsumers' % ACTION_PREFIX:
result = {
'Consumers': [
c for c in STREAM_CONSUMERS
if c.get('StreamARN') == data.get('StreamARN')
]
}
return result
elif action == '%s.DescribeStreamConsumer' % ACTION_PREFIX:
consumer_arn = data.get('ConsumerARN') or data['ConsumerName']
consumer_name = data.get('ConsumerName') or data['ConsumerARN']
result = {
'ConsumerDescription': {
'ConsumerARN': consumer_arn,
# 'ConsumerCreationTimestamp': number,
'ConsumerName': consumer_name,
'ConsumerStatus': 'ACTIVE',
'StreamARN': data.get('StreamARN')
}
}
return result
if random.random() < config.KINESIS_ERROR_PROBABILITY:
action = headers.get('X-Amz-Target')
if action in [ACTION_PUT_RECORD, ACTION_PUT_RECORDS]:
return kinesis_error_response(data, action)
return True
def _execute(self, func_arn, func_details, event, context=None, version=None):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = func_details.envvars.copy()
# configure USE_SSL in environment
if config.USE_SSL:
environment['USE_SSL'] = '1'
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
docker_host = config.DOCKER_HOST_FROM_CONTAINER
environment['HOSTNAME'] = docker_host
environment['LOCALSTACK_HOSTNAME'] = docker_host
if context:
environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name
environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version
environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn
# custom command to execute in the container
command = ''
# if running a Java Lambda, set up classpath arguments
if runtime == LAMBDA_RUNTIME_JAVA8:
java_opts = Util.get_java_opts()
stdin = None
# copy executor jar into temp directory
target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR))
if not os.path.exists(target_file):
cp_r(LAMBDA_EXECUTOR_JAR, target_file)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body)
classpath = Util.get_java_classpath(target_file)
command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" %
(taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE))
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd)
# lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
LOG.debug('Running lambda cmd: %s' % cmd)
result, log_output = self.run_lambda_executor(cmd, stdin, environment)
log_formatted = log_output.strip().replace('\n', '\n> ')
LOG.debug('Lambda %s result / log output:\n%s\n>%s' % (func_arn, result.strip(), log_formatted))
return result, log_output
def get_resource_dependencies(resource_id, resource, resources):
result = {}
dumped = json.dumps(common.json_safe(resource))
for other_id, other in iteritems(resources):
if resource != other:
# TODO: traverse dict instead of doing string search
search1 = '{"Ref": "%s"}' % other_id
search2 = '{"Fn::GetAtt": ["%s", ' % other_id
if search1 in dumped or search2 in dumped:
result[other_id] = other
# print('deps %s %s' % (resource_id, len(result)))
return result
def extract_path_params(path, extracted_path):
tokenized_extracted_path = tokenize_path(extracted_path)
# Looks for '{' in the tokenized extracted path
path_params_list = [(i, v) for i, v in enumerate(tokenized_extracted_path) if '{' in v]
tokenized_path = tokenize_path(path)
path_params = {}
for param in path_params_list:
path_param_name = param[1][1:-1].encode('utf-8')
path_param_position = param[0]
path_params[path_param_name] = tokenized_path[path_param_position]
path_params = common.json_safe(path_params)
return path_params
def test_non_ascii_chars(self):
dynamodb = aws_stack.connect_to_resource('dynamodb')
testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME, partition_key=PARTITION_KEY)
table = dynamodb.Table(TEST_DDB_TABLE_NAME)
# write some items containing non-ASCII characters
items = {
'id1': {PARTITION_KEY: 'id1', 'data': 'foobar123 ✓'},
'id2': {PARTITION_KEY: 'id2', 'data': 'foobar123 £'},
'id3': {PARTITION_KEY: 'id3', 'data': 'foobar123 ¢'}
}
for k, item in items.items():
table.put_item(Item=item)
for item_id in items.keys():
item = table.get_item(Key={PARTITION_KEY: item_id})['Item']
# need to fix up the JSON and convert str to unicode for Python 2
item1 = json_safe(item)
item2 = json_safe(items[item_id])
assert item1 == item2
def get_resource_dependencies(resource_id, resource, resources):
result = {}
dumped = json.dumps(common.json_safe(resource))
for other_id, other in iteritems(resources):
if resource != other:
# TODO: traverse dict instead of doing string search
search1 = '{"Ref": "%s"}' % other_id
search2 = '{"Fn::GetAtt": ["%s", ' % other_id
if search1 in dumped or search2 in dumped:
result[other_id] = other
if other_id in resource.get('DependsOn', []):
result[other_id] = other
return result
def get_resource_dependencies(resource_id, resource, resources):
result = {}
dumped = json.dumps(common.json_safe(resource))
dependencies = resource.get('DependsOn', [])
for other_id, other in resources.items():
if resource != other:
# TODO: traverse dict instead of doing string search
search1 = '{"Ref": "%s"}' % other_id
search2 = '{"Fn::GetAtt": ["%s", ' % other_id
if search1 in dumped or search2 in dumped:
result[other_id] = other
if other_id in dependencies:
result[other_id] = other
return result
def extract_path_params(path, extracted_path):
tokenized_extracted_path = tokenize_path(extracted_path)
# Looks for '{' in the tokenized extracted path
path_params_list = [(i, v) for i, v in enumerate(tokenized_extracted_path) if '{' in v]
tokenized_path = tokenize_path(path)
path_params = {}
for param in path_params_list:
path_param_name = param[1][1:-1].encode('utf-8')
path_param_position = param[0]
if path_param_name.endswith(b'+'):
path_params[path_param_name] = '/'.join(tokenized_path[path_param_position:])
else:
path_params[path_param_name] = tokenized_path[path_param_position]
path_params = common.json_safe(path_params)
return path_params
def requests_response_xml(action, response, xmlns=None, service=None):
xmlns = xmlns or 'http://%s.amazonaws.com/doc/2010-03-31/' % service
response = json_safe(response)
response = {'{action}Result'.format(action=action): response}
response = xmltodict.unparse(response)
if response.startswith('<?xml'):
response = re.sub(r'<\?xml [^\?]+\?>', '', response)
result = ("""
<{action}Response xmlns="{xmlns}">
{response}
</{action}Response>
""").strip()
result = result.format(action=action, xmlns=xmlns, response=response)
result = requests_response(result)
return result
def forward_request(self, method, path, data, **kwargs):
response = Response()
response.status_code = 200
response._content = '{}'
try:
if path == API_PATH_SERVERS:
if method == 'POST':
start_api_server_locally(json.loads(to_str(data)))
elif method == 'GET':
response._content = json.dumps(json_safe(API_SERVERS))
except Exception as e:
LOG.error('Unable to process request: %s' % e)
response.status_code = 500
response._content = str(e)
return response
def extract_path_params(path, extracted_path):
tokenized_extracted_path = tokenize_path(extracted_path)
# Looks for '{' in the tokenized extracted path
path_params_list = [(i, v) for i, v in enumerate(tokenized_extracted_path) if "{" in v]
tokenized_path = tokenize_path(path)
path_params = {}
for param in path_params_list:
path_param_name = param[1][1:-1].encode("utf-8")
path_param_position = param[0]
if path_param_name.endswith(b"+"):
path_params[path_param_name] = "/".join(tokenized_path[path_param_position:])
else:
path_params[path_param_name] = tokenized_path[path_param_position]
path_params = common.json_safe(path_params)
return path_params
def requests_response_xml(action, response, xmlns=None, service=None, memberize=True):
xmlns = xmlns or "http://%s.amazonaws.com/doc/2010-03-31/" % service
response = json_safe(response)
response = {"{action}Result".format(action=action): response}
response = ET.tostring(to_xml(response, memberize=memberize), short_empty_elements=True)
response = to_str(response)
result = (
"""
<{action}Response xmlns="{xmlns}">
{response}
</{action}Response>
"""
).strip()
result = result.format(action=action, xmlns=xmlns, response=response)
result = requests_response(result)
return result
def execute_javascript_lambda(self, event, context, main_file, func_details=None):
handler = func_details.handler
function = handler.split(".")[-1]
event_json_string = "%s" % (json.dumps(json_safe(event)) if event else "{}")
context_json_string = "%s" % (json.dumps(context.__dict__) if context else "{}")
cmd = (
"node -e 'require(\"%s\").%s(%s,%s).then(r => process.stdout.write(JSON.stringify(r)))'"
% (
main_file,
function,
event_json_string,
context_json_string,
)
)
LOG.info(cmd)
result = self._execute_in_custom_runtime(cmd, func_details=func_details)
return result
def send_events():
yield convert_to_binary_event_payload('', event_type='initial-response')
iter = iterator
# TODO: find better way to run loop up to max 5 minutes (until connection terminates)!
for i in range(5 * 60):
result = kinesis.get_records(ShardIterator=iter)
iter = result.get('NextShardIterator')
records = result.get('Records', [])
for record in records:
record['ApproximateArrivalTimestamp'] = record['ApproximateArrivalTimestamp'].timestamp()
if data_needs_encoding:
record['Data'] = base64.b64encode(record['Data'])
record['Data'] = to_str(record['Data'])
if not records:
time.sleep(1)
continue
result = json.dumps({'Records': json_safe(records)})
yield convert_to_binary_event_payload(result, event_type='SubscribeToShardEvent')
def send_events():
yield convert_to_binary_event_payload("",
event_type="initial-response")
iter = iterator
last_sequence_number = starting_sequence_number
# TODO: find better way to run loop up to max 5 minutes (until connection terminates)!
for i in range(5 * 60):
result = None
try:
result = kinesis.get_records(ShardIterator=iter)
except Exception as e:
if "ResourceNotFoundException" in str(e):
LOG.debug(
'Kinesis stream "%s" has been deleted, closing shard subscriber',
stream_name,
)
return
raise
iter = result.get("NextShardIterator")
records = result.get("Records", [])
for record in records:
record["ApproximateArrivalTimestamp"] = record[
"ApproximateArrivalTimestamp"].timestamp()
# boto3 automatically decodes records in get_records(), so we must re-encode
record["Data"] = to_str(base64.b64encode(record["Data"]))
last_sequence_number = record["SequenceNumber"]
if not records:
time.sleep(1)
continue
response = {
"ChildShards": [],
"ContinuationSequenceNumber": last_sequence_number,
"MillisBehindLatest": 0,
"Records": json_safe(records),
}
result = json.dumps(response)
yield convert_to_binary_event_payload(
result, event_type="SubscribeToShardEvent")
def modify_and_forward(method=None,
path=None,
data_bytes=None,
headers=None,
forward_base_url=None,
listeners=None,
request_handler=None,
client_address=None,
server_address=None):
listeners = GenericProxyHandler.DEFAULT_LISTENERS + (listeners or [])
listeners = [lis for lis in listeners if lis]
data = data_bytes
def is_full_url(url):
return re.match(r'[a-zA-Z]+://.+', url)
if is_full_url(path):
path = path.split('://', 1)[1]
path = '/%s' % (path.split('/', 1)[1] if '/' in path else '')
proxy_url = '%s%s' % (forward_base_url, path)
for listener in listeners:
proxy_url = listener.get_forward_url(method, path, data,
headers) or proxy_url
target_url = path
if not is_full_url(target_url):
target_url = '%s%s' % (forward_base_url, target_url)
# update original "Host" header (moto s3 relies on this behavior)
if not headers.get('Host'):
headers['host'] = urlparse(target_url).netloc
if 'localhost.atlassian.io' in headers.get('Host'):
headers['host'] = 'localhost'
headers['X-Forwarded-For'] = build_x_forwarded_for(headers, client_address,
server_address)
response = None
modified_request = None
# update listener (pre-invocation)
for listener in listeners:
listener_result = listener.forward_request(method=method,
path=path,
data=data,
headers=headers)
if isinstance(listener_result, Response):
response = listener_result
break
if isinstance(listener_result, LambdaResponse):
response = listener_result
break
if isinstance(listener_result, dict):
response = Response()
response._content = json.dumps(json_safe(listener_result))
response.headers['Content-Type'] = APPLICATION_JSON
response.status_code = 200
break
elif isinstance(listener_result, Request):
modified_request = listener_result
data = modified_request.data
headers = modified_request.headers
break
elif listener_result is not True:
# get status code from response, or use Bad Gateway status code
code = listener_result if isinstance(listener_result, int) else 503
response = Response()
response.status_code = code
response._content = ''
response.headers['Content-Length'] = '0'
append_cors_headers(response)
return response
# perform the actual invocation of the backend service
if response is None:
headers['Connection'] = headers.get('Connection') or 'close'
data_to_send = data_bytes
request_url = proxy_url
if modified_request:
if modified_request.url:
request_url = '%s%s' % (forward_base_url, modified_request.url)
data_to_send = modified_request.data
requests_method = getattr(requests, method.lower())
response = requests_method(request_url,
data=data_to_send,
headers=headers,
stream=True)
# prevent requests from processing response body (e.g., to pass-through gzip encoded content unmodified)
pass_raw = ((hasattr(response, '_content_consumed')
and not response._content_consumed)
or response.headers.get('content-encoding') in ['gzip'])
if pass_raw and response.raw:
new_content = response.raw.read()
if new_content:
response._content = new_content
# update listener (post-invocation)
if listeners:
update_listener = listeners[-1]
kwargs = {
'method': method,
'path': path,
'data': data_bytes,
'headers': headers,
'response': response
}
if 'request_handler' in inspect.getargspec(
update_listener.return_response)[0]:
# some listeners (e.g., sqs_listener.py) require additional details like the original
# request port, hence we pass in a reference to this request handler as well.
kwargs['request_handler'] = request_handler
updated_response = update_listener.return_response(**kwargs)
if isinstance(updated_response, Response):
response = updated_response
# allow pre-flight CORS headers by default
append_cors_headers(response)
return response
def invoke_rest_api(api_id,
stage,
method,
invocation_path,
data,
headers,
path=None):
path = path or invocation_path
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
# run gateway authorizers for this request
authorize_invocation(api_id, headers)
path_map = helpers.get_rest_api_paths(rest_api_id=api_id)
try:
extracted_path, resource = get_resource_for_path(path=relative_path,
path_map=path_map)
except Exception:
return make_error_response('Unable to find path %s' % path, 404)
api_key_required = resource.get('resourceMethods',
{}).get(method, {}).get('apiKeyRequired')
if not is_api_key_valid(api_key_required, headers, stage):
return make_error_response('Access denied - invalid API key', 403)
integrations = resource.get('resourceMethods', {})
integration = integrations.get(method, {})
if not integration:
integration = integrations.get('ANY', {})
integration = integration.get('methodIntegration')
if not integration:
if method == 'OPTIONS' and 'Origin' in headers:
# default to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
return make_error_response(
'Unable to find integration for path %s' % path, 404)
uri = integration.get('uri') or ''
integration_type = integration['type'].upper()
if uri.startswith('arn:aws:apigateway:') and ':lambda:path' in uri:
if integration_type in ['AWS', 'AWS_PROXY']:
func_arn = uri.split(':lambda:path')[1].split(
'functions/')[1].split('/invocations')[0]
data_str = json.dumps(data) if isinstance(data,
(dict,
list)) else to_str(data)
account_id = uri.split(':lambda:path')[1].split(
':function:')[0].split(':')[-1]
source_ip = headers['X-Forwarded-For'].split(',')[-2]
try:
path_params = extract_path_params(
path=relative_path, extracted_path=extracted_path)
except Exception:
path_params = {}
# apply custom request template
data_str = apply_template(integration,
'request',
data_str,
path_params=path_params,
query_params=query_string_params,
headers=headers)
# Sample request context:
# https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-create-api-as-simple-proxy-for-lambda.html#api-gateway-create-api-as-simple-proxy-for-lambda-test
request_context = {
# adding stage to the request context path.
# https://github.com/localstack/localstack/issues/2210
'path': '/' + stage + relative_path,
'accountId': account_id,
'resourceId': resource.get('id'),
'stage': stage,
'identity': {
'accountId': account_id,
'sourceIp': source_ip,
'userAgent': headers['User-Agent'],
},
'httpMethod': method,
'protocol': 'HTTP/1.1',
'requestTime': datetime.datetime.utcnow(),
'requestTimeEpoch': int(time.time() * 1000),
}
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
data_str,
stage,
api_id,
headers,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=path,
request_context=request_context)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = result if isinstance(
result, dict) else json.loads(str(result or '{}'))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get(
'statusCode', 200))
parsed_headers = parsed_result.get('headers', {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
if isinstance(parsed_result['body'], dict):
response._content = json.dumps(parsed_result['body'])
else:
response._content = to_bytes(parsed_result['body'])
except Exception:
response._content = '{}'
update_content_length(response)
response.multi_value_headers = parsed_result.get(
'multiValueHeaders') or {}
# apply custom response template
response._content = apply_template(integration, 'response',
response._content)
response.headers['Content-Length'] = str(
len(response.content or ''))
return response
msg = 'API Gateway AWS integration action URI "%s", method "%s" not yet implemented' % (
uri, method)
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration_type == 'AWS':
if 'kinesis:action/' in uri:
if uri.endswith('kinesis:action/PutRecords'):
target = kinesis_listener.ACTION_PUT_RECORDS
if uri.endswith('kinesis:action/ListStreams'):
target = kinesis_listener.ACTION_LIST_STREAMS
template = integration['requestTemplates'][APPLICATION_JSON]
new_request = aws_stack.render_velocity_template(template, data)
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(service='kinesis')
headers['X-Amz-Target'] = target
result = common.make_http_request(url=TEST_KINESIS_URL,
method='POST',
data=new_request,
headers=headers)
# TODO apply response template..?
return result
if method == 'POST':
if uri.startswith('arn:aws:apigateway:') and ':sqs:path' in uri:
template = integration['requestTemplates'][APPLICATION_JSON]
account_id, queue = uri.split('/')[-2:]
region_name = uri.split(':')[3]
new_request = '%s&QueueName=%s' % (
aws_stack.render_velocity_template(template, data), queue)
headers = aws_stack.mock_aws_request_headers(
service='sqs', region_name=region_name)
url = urljoin(TEST_SQS_URL,
'%s/%s' % (TEST_AWS_ACCOUNT_ID, queue))
result = common.make_http_request(url,
method='POST',
headers=headers,
data=new_request)
return result
msg = 'API Gateway AWS integration action URI "%s", method "%s" not yet implemented' % (
uri, method)
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration_type == 'AWS_PROXY':
if uri.startswith('arn:aws:apigateway:') and ':dynamodb:action' in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(':dynamodb:action')[1].split('&Table=')[1]
action = uri.split(':dynamodb:action')[1].split('&Table=')[0]
if 'PutItem' in action and method == 'PUT':
response_template = path_map.get(relative_path, {}).get('resourceMethods', {})\
.get(method, {}).get('methodIntegration', {}).\
get('integrationResponses', {}).get('200', {}).get('responseTemplates', {})\
.get('application/json', None)
if response_template is None:
msg = 'Invalid response template defined in integration response.'
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template['TableName'] != table_name:
msg = 'Invalid table name specified in integration response template.'
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource('dynamodb')
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template['Item'].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(
event_data, headers=aws_stack.mock_aws_request_headers())
return response
else:
msg = 'API Gateway action uri "%s" not yet implemented' % uri
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration_type in ['HTTP_PROXY', 'HTTP']:
function = getattr(requests, method.lower())
# apply custom request template
data = apply_template(integration, 'request', data)
if isinstance(data, dict):
data = json.dumps(data)
result = function(integration['uri'], data=data, headers=headers)
# apply custom response template
data = apply_template(integration, 'response', data)
return result
elif integration_type == 'MOCK':
# TODO: add logic for MOCK responses
pass
if method == 'OPTIONS':
# fall back to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
msg = (
'API Gateway integration type "%s", method "%s", URI "%s" not yet implemented'
% (integration['type'], method, integration.get('uri')))
LOGGER.warning(msg)
return make_error_response(msg, 404)
def forward_request(self, method, path, data, headers):
data = data and json.loads(to_str(data))
# Paths to match
regex2 = r'^/restapis/([A-Za-z0-9_\-]+)/([A-Za-z0-9_\-]+)/%s/(.*)$' % PATH_USER_REQUEST
if re.match(regex2, path):
search_match = re.search(regex2, path)
api_id = search_match.group(1)
relative_path = '/%s' % search_match.group(3)
try:
integration = aws_stack.get_apigateway_integration(api_id, method, path=relative_path)
assert integration
except Exception:
# if we have no exact match, try to find an API resource that contains path parameters
path_map = get_rest_api_paths(rest_api_id=api_id)
try:
extracted_path, resource = get_resource_for_path(path=relative_path, path_map=path_map)
except Exception:
return make_error('Unable to find path %s' % path, 404)
integrations = resource.get('resourceMethods', {})
integration = integrations.get(method, {})
integration = integration.get('methodIntegration')
if not integration:
if method == 'OPTIONS' and 'Origin' in headers:
# default to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
return make_error('Unable to find integration for path %s' % path, 404)
uri = integration.get('uri')
if method == 'POST' and integration['type'] == 'AWS':
if uri.endswith('kinesis:action/PutRecords'):
template = integration['requestTemplates'][APPLICATION_JSON]
new_request = aws_stack.render_velocity_template(template, data)
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(service='kinesis')
headers['X-Amz-Target'] = kinesis_listener.ACTION_PUT_RECORDS
result = common.make_http_request(url=TEST_KINESIS_URL,
method='POST', data=new_request, headers=headers)
return result
else:
msg = 'API Gateway action uri "%s" not yet implemented' % uri
LOGGER.warning(msg)
return make_error(msg, 404)
elif integration['type'] == 'AWS_PROXY':
if uri.startswith('arn:aws:apigateway:') and ':lambda:path' in uri:
func_arn = uri.split(':lambda:path')[1].split('functions/')[1].split('/invocations')[0]
data_str = json.dumps(data) if isinstance(data, dict) else data
try:
path_params = extract_path_params(path=relative_path, extracted_path=extracted_path)
except Exception:
path_params = {}
result = lambda_api.process_apigateway_invocation(func_arn, relative_path, data_str,
headers, path_params=path_params, method=method, resource_path=path)
if isinstance(result, FlaskResponse):
return flask_to_requests_response(result)
response = Response()
parsed_result = result if isinstance(result, dict) else json.loads(result)
parsed_result = common.json_safe(parsed_result)
response.status_code = int(parsed_result.get('statusCode', 200))
response.headers.update(parsed_result.get('headers', {}))
try:
if isinstance(parsed_result['body'], dict):
response._content = json.dumps(parsed_result['body'])
else:
response._content = parsed_result['body']
except Exception:
response._content = '{}'
return response
else:
msg = 'API Gateway action uri "%s" not yet implemented' % uri
LOGGER.warning(msg)
return make_error(msg, 404)
elif integration['type'] == 'HTTP':
function = getattr(requests, method.lower())
if isinstance(data, dict):
data = json.dumps(data)
result = function(integration['uri'], data=data, headers=headers)
return result
else:
msg = ('API Gateway integration type "%s" for method "%s" not yet implemented' %
(integration['type'], method))
LOGGER.warning(msg)
return make_error(msg, 404)
return 200
if re.match(PATH_REGEX_AUTHORIZERS, path):
return handle_authorizers(method, path, data, headers)
return True