def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = json.loads(to_str(data))
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM if action == ACTION_CREATE_STREAM
else event_publisher.EVENT_KINESIS_DELETE_STREAM)
event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(data.get('StreamName'))})
elif action == ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber': response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
def delete_function(function):
""" Delete an existing function
---
operationId: 'deleteFunction'
parameters:
- name: 'request'
in: body
"""
arn = func_arn(function)
# Stop/remove any containers that this arn uses.
LAMBDA_EXECUTOR.cleanup(arn)
try:
arn_to_lambda.pop(arn)
except KeyError:
return error_response('Function does not exist: %s' % function, 404, error_type='ResourceNotFoundException')
event_publisher.fire_event(event_publisher.EVENT_LAMBDA_DELETE_FUNC,
payload={'n': event_publisher.get_hash(function)})
i = 0
while i < len(event_source_mappings):
mapping = event_source_mappings[i]
if mapping['FunctionArn'] == arn:
del event_source_mappings[i]
i -= 1
i += 1
result = {}
return jsonify(result)
def delete_stream(stream_name):
stream = DELIVERY_STREAMS.pop(stream_name, {})
if not stream:
return error_not_found(stream_name)
# record event
event_publisher.fire_event(
event_publisher.EVENT_FIREHOSE_DELETE_STREAM,
payload={'n': event_publisher.get_hash(stream_name)})
return {}
def publish_event(time_before, result, kwargs):
event_publisher.fire_event(event_publisher.EVENT_LAMBDA_INVOKE_FUNC,
payload={
'f':
event_publisher.get_hash(
_func_name(kwargs)),
'd':
now_utc() - time_before,
'r':
result[0]
})
def forward_request(self, method, path, data, headers):
if method == 'OPTIONS':
return 200
req_data = None
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
req_data = dict([(k, v[0]) for k, v in req_data.items()])
action = req_data.get('Action')
if action == 'CreateStack':
stack_name = req_data.get('StackName')
event_publisher.fire_event(
event_publisher.EVENT_CLOUDFORMATION_CREATE_STACK,
payload={'n': event_publisher.get_hash(stack_name)})
if action == 'DescribeStackEvents':
# fix an issue where moto cannot handle ARNs as stack names (or missing names)
stack_name = req_data.get('StackName')
run_fix = not stack_name
if stack_name:
if stack_name.startswith('arn:aws:cloudformation'):
run_fix = True
stack_name = re.sub(
r'arn:aws:cloudformation:[^:]+:[^:]+:stack/([^/]+)(/.+)?',
r'\1', stack_name)
if run_fix:
stack_names = [
stack_name
] if stack_name else self._list_stack_names()
client = aws_stack.connect_to_service('cloudformation')
events = []
for stack_name in stack_names:
tmp = client.describe_stack_events(
StackName=stack_name)['StackEvents'][:1]
events.extend(tmp)
events = [{'member': e} for e in events]
response_content = '<StackEvents>%s</StackEvents>' % obj_to_xml(
events)
return make_response('DescribeStackEvents',
response_content)
if req_data:
if action == 'ValidateTemplate':
return validate_template(req_data)
if action == 'CreateStack':
modified_request = transform_template(req_data)
if modified_request:
req_data.pop('TemplateURL', None)
req_data['TemplateBody'] = json.dumps(modified_request)
data = urlparse.urlencode(req_data, doseq=True)
return Request(data=data, headers=headers, method=method)
return True
def do_start_infra(asynchronous, apis, is_in_docker):
event_publisher.fire_event(event_publisher.EVENT_START_INFRA,
{'d': is_in_docker and 1 or 0, 'c': in_ci() and 1 or 0})
# set up logging
setup_logging()
# prepare APIs
apis = canonicalize_api_names(apis)
@log_duration()
def prepare_environment():
# set environment
os.environ['AWS_REGION'] = config.DEFAULT_REGION
os.environ['ENV'] = ENV_DEV
# register signal handlers
if not is_local_test_mode():
register_signal_handlers()
# make sure AWS credentials are configured, otherwise boto3 bails on us
check_aws_credentials()
@log_duration()
def prepare_installation():
# install libs if not present
install.install_components(apis)
@log_duration()
def start_api_services():
# Some services take a bit to come up
sleep_time = 5
# start services
thread = None
# loop through plugins and start each service
for name, plugin in SERVICE_PLUGINS.items():
if plugin.is_enabled(api_names=apis):
record_service_health(name, 'starting')
t1 = plugin.start(asynchronous=True)
thread = thread or t1
time.sleep(sleep_time)
# ensure that all infra components are up and running
check_infra(apis=apis)
# restore persisted data
persistence.restore_persisted_data(apis=apis)
return thread
prepare_environment()
prepare_installation()
thread = start_api_services()
print('Ready.')
sys.stdout.flush()
return thread
def stop_infra():
if common.INFRA_STOPPED:
return
common.INFRA_STOPPED = True
event_publisher.fire_event(event_publisher.EVENT_STOP_INFRA)
generic_proxy.QUIET = True
common.cleanup(files=True, quiet=True)
common.cleanup_resources()
lambda_api.cleanup()
time.sleep(2)
def return_response(self, method, path, data, headers, response):
data = json.loads(to_str(data or '{}'))
name = data.get('name') or (data.get('stateMachineArn') or '').split(':')[-1]
target = headers.get('X-Amz-Target')
# publish event
if target == 'AWSStepFunctions.CreateStateMachine':
event_publisher.fire_event(event_publisher.EVENT_STEPFUNCTIONS_CREATE_SM,
payload={'m': event_publisher.get_hash(name)})
elif target == 'AWSStepFunctions.DeleteStateMachine':
event_publisher.fire_event(event_publisher.EVENT_STEPFUNCTIONS_DELETE_SM,
payload={'m': event_publisher.get_hash(name)})
def delete_domain(domain_name):
if domain_name not in ES_DOMAINS:
return error_response(error_type='ResourceNotFoundException')
result = get_domain_status(domain_name, deleted=True)
ES_DOMAINS.pop(domain_name)
if not ES_DOMAINS:
cleanup_elasticsearch_instance()
# record event
event_publisher.fire_event(
event_publisher.EVENT_ES_DELETE_DOMAIN,
payload={'n': event_publisher.get_hash(domain_name)})
return jsonify(result)
def create_domain():
data = json.loads(to_str(request.data))
domain_name = data['DomainName']
if domain_name in ES_DOMAINS:
return error_response(error_type='ResourceAlreadyExistsException')
ES_DOMAINS[domain_name] = data
result = get_domain_status(domain_name)
# record event
event_publisher.fire_event(
event_publisher.EVENT_ES_CREATE_DOMAIN,
payload={'n': event_publisher.get_hash(domain_name)})
return jsonify(result)
def create_domain(
self,
context: RequestContext,
domain_name: DomainName,
engine_version: VersionString = None,
cluster_config: ClusterConfig = None,
ebs_options: EBSOptions = None,
access_policies: PolicyDocument = None,
snapshot_options: SnapshotOptions = None,
vpc_options: VPCOptions = None,
cognito_options: CognitoOptions = None,
encryption_at_rest_options: EncryptionAtRestOptions = None,
node_to_node_encryption_options: NodeToNodeEncryptionOptions = None,
advanced_options: AdvancedOptions = None,
log_publishing_options: LogPublishingOptions = None,
domain_endpoint_options: DomainEndpointOptions = None,
advanced_security_options: AdvancedSecurityOptionsInput = None,
tag_list: TagList = None,
auto_tune_options: AutoTuneOptionsInput = None,
) -> CreateDomainResponse:
region = OpenSearchServiceBackend.get()
with _domain_mutex:
if domain_name in region.opensearch_domains:
raise ResourceAlreadyExistsException(
f"domain {domain_name} already exists in region {region.name}"
)
domain_key = DomainKey(
domain_name=domain_name,
region=context.region,
account=context.account_id,
)
# "create" domain data
region.opensearch_domains[domain_name] = get_domain_status(
domain_key)
# lazy-init the cluster (sets the Endpoint and Processing flag of the domain status)
# TODO handle additional parameters (cluster config,...)
create_cluster(domain_key, engine_version, domain_endpoint_options)
# set the tags
self.add_tags(context, domain_key.arn, tag_list)
# get the (updated) status
status = get_domain_status(domain_key)
# record event
event_publisher.fire_event(
event_publisher.EVENT_OPENSEARCH_CREATE_DOMAIN,
payload={"n": event_publisher.get_hash(domain_name)},
)
return CreateDomainResponse(DomainStatus=status)
def delete_rest_api(self, context: RequestContext, rest_api_id: String) -> None:
try:
call_moto(context)
except KeyError as e:
# moto raises a key error if we're trying to delete an API that doesn't exist
raise NotFoundException(
f"Invalid API identifier specified {context.account_id}:{rest_api_id}"
) from e
event_publisher.fire_event(
event_publisher.EVENT_APIGW_DELETE_API,
payload={"a": event_publisher.get_hash(rest_api_id)},
)
def delete_stream(stream_name: str) -> Dict:
region = FirehoseBackend.get()
stream = region.delivery_streams.pop(stream_name, {})
if not stream:
return error_not_found(stream_name)
# record event
event_publisher.fire_event(
event_publisher.EVENT_FIREHOSE_DELETE_STREAM,
payload={"n": event_publisher.get_hash(stream_name)},
)
return {}
def return_response(self, method, path, data, headers, response):
# persist requests to disk
super(ProxyListenerSNS, self).return_response(method, path, data,
headers, response)
if method == "POST" and path == "/":
# convert account IDs in ARNs
data = aws_stack.fix_account_id_in_arns(data,
colon_delimiter="%3A")
aws_stack.fix_account_id_in_arns(response)
# remove "None" strings from result
search = r"<entry><key>[^<]+</key>\s*<value>\s*None\s*</[^>]+>\s*</entry>"
response_regex_replace(response, search, "")
# parse request and extract data
req_data = parse_qs(to_str(data))
req_action = req_data["Action"][0]
if req_action == "Subscribe" and response.status_code < 400:
response_data = xmltodict.parse(response.content)
topic_arn = (req_data.get("TargetArn")
or req_data.get("TopicArn"))[0]
filter_policy = (req_data.get("FilterPolicy") or [None])[0]
attributes = get_subscribe_attributes(req_data)
sub_arn = response_data["SubscribeResponse"][
"SubscribeResult"]["SubscriptionArn"]
do_subscribe(
topic_arn,
req_data["Endpoint"][0],
req_data["Protocol"][0],
sub_arn,
attributes,
filter_policy,
)
if req_action == "CreateTopic" and response.status_code < 400:
response_data = xmltodict.parse(response.content)
topic_arn = response_data["CreateTopicResponse"][
"CreateTopicResult"]["TopicArn"]
# publish event
event_publisher.fire_event(
event_publisher.EVENT_SNS_CREATE_TOPIC,
payload={"t": event_publisher.get_hash(topic_arn)},
)
if req_action == "DeleteTopic" and response.status_code < 400:
# publish event
topic_arn = (req_data.get("TargetArn")
or req_data.get("TopicArn"))[0]
event_publisher.fire_event(
event_publisher.EVENT_SNS_DELETE_TOPIC,
payload={"t": event_publisher.get_hash(topic_arn)},
)
def return_response(self, method, path, data, headers, response):
parsed = urlparse.urlparse(path)
# TODO: consider the case of hostname-based (as opposed to path-based) bucket addressing
bucket_name = parsed.path.split('/')[1]
# POST requests to S3 may include a success_action_redirect field,
# which should be used to redirect a client to a new location.
if method == 'POST':
key, redirect_url = find_multipart_redirect_url(data, headers)
if key and redirect_url:
response.status_code = 303
response.headers['Location'] = expand_redirect_url(redirect_url, key, bucket_name)
LOGGER.debug('S3 POST {} to {}'.format(response.status_code, response.headers['Location']))
# get subscribers and send bucket notifications
if method in ('PUT', 'DELETE') and '/' in path[1:]:
# check if this is an actual put object request, because it could also be
# a put bucket request with a path like this: /bucket_name/
if len(path[1:].split('/')[1]) > 0:
parts = parsed.path[1:].split('/', 1)
# ignore bucket notification configuration requests
if parsed.query != 'notification':
object_path = parts[1] if parts[1][0] == '/' else '/%s' % parts[1]
send_notifications(method, bucket_name, object_path)
# publish event for creation/deletion of buckets:
if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0):
event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT'
else event_publisher.EVENT_S3_DELETE_BUCKET)
event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(bucket_name)})
# append CORS headers to response
if response:
append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response)
response_content_str = None
try:
response_content_str = to_str(response._content)
except Exception:
pass
# we need to un-pretty-print the XML, otherwise we run into this issue with Spark:
# https://github.com/jserver/mock-s3/pull/9/files
# https://github.com/localstack/localstack/issues/183
if response_content_str and response_content_str.startswith('<'):
is_bytes = isinstance(response._content, six.binary_type)
response._content = re.sub(r'>\n\s*<', '><', response_content_str, flags=re.MULTILINE)
if is_bytes:
response._content = to_bytes(response._content)
response.headers['content-length'] = len(response._content)
def create_function():
""" Create new function
---
operationId: 'createFunction'
parameters:
- name: 'request'
in: body
"""
arn = 'n/a'
try:
data = json.loads(to_str(request.data))
lambda_name = data['FunctionName']
event_publisher.fire_event(
event_publisher.EVENT_LAMBDA_CREATE_FUNC,
payload={'n': event_publisher.get_hash(lambda_name)})
arn = func_arn(lambda_name)
if arn in arn_to_lambda:
return error_response('Function already exist: %s' % lambda_name,
409,
error_type='ResourceConflictException')
arn_to_lambda[arn] = func_details = LambdaFunction(arn)
func_details.versions = {'$LATEST': {'RevisionId': str(uuid.uuid4())}}
func_details.last_modified = isoformat_milliseconds(
datetime.utcnow()) + '+0000'
func_details.description = data.get('Description', '')
func_details.handler = data['Handler']
func_details.runtime = data['Runtime']
func_details.envvars = data.get('Environment', {}).get('Variables', {})
func_details.tags = data.get('Tags', {})
func_details.timeout = data.get('Timeout', LAMBDA_DEFAULT_TIMEOUT)
func_details.role = data['Role']
func_details.memory_size = data.get('MemorySize')
func_details.code = data['Code']
result = set_function_code(func_details.code, lambda_name)
if isinstance(result, Response):
del arn_to_lambda[arn]
return result
# remove content from code attribute, if present
func_details.code.pop('ZipFile', None)
# prepare result
result.update(format_func_details(func_details))
if data.get('Publish', False):
result['Version'] = publish_new_function_version(arn)['Version']
return jsonify(result or {})
except Exception as e:
arn_to_lambda.pop(arn, None)
if isinstance(e, ClientError):
return e.get_response()
return error_response('Unknown error: %s %s' %
(e, traceback.format_exc()))
def create_function():
""" Create new function
---
operationId: 'createFunction'
parameters:
- name: 'request'
in: body
"""
arn = 'n/a'
try:
data = json.loads(to_str(request.data))
lambda_name = data['FunctionName']
event_publisher.fire_event(event_publisher.EVENT_LAMBDA_CREATE_FUNC,
payload={'n': event_publisher.get_hash(lambda_name)})
arn = func_arn(lambda_name)
if arn in arn_to_lambda:
return error_response('Function already exist: %s' %
lambda_name, 409, error_type='ResourceConflictException')
arn_to_lambda[arn] = func_details = LambdaFunction(arn)
func_details.versions = {'$LATEST': {'CodeSize': 50}}
func_details.handler = data['Handler']
func_details.runtime = data['Runtime']
func_details.envvars = data.get('Environment', {}).get('Variables', {})
func_details.tags = data.get('Tags', {})
func_details.timeout = data.get('Timeout')
result = set_function_code(data['Code'], lambda_name)
if isinstance(result, Response):
del arn_to_lambda[arn]
return result
result.update({
'DeadLetterConfig': data.get('DeadLetterConfig'),
'Description': data.get('Description'),
'Environment': {'Error': {}, 'Variables': func_details.envvars},
'FunctionArn': arn,
'FunctionName': lambda_name,
'Handler': func_details.handler,
'MemorySize': data.get('MemorySize'),
'Role': data.get('Role'),
'Runtime': func_details.runtime,
'Timeout': data.get('Timeout'),
'TracingConfig': {},
'VpcConfig': {'SecurityGroupIds': [None], 'SubnetIds': [None], 'VpcId': None}
})
if data.get('Publish', False):
result['Version'] = publish_new_function_version(arn)['Version']
return jsonify(result or {})
except Exception as e:
arn_to_lambda.pop(arn, None)
return error_response('Unknown error: %s %s' % (e, traceback.format_exc()))
def create_function():
""" Create new function
---
operationId: 'createFunction'
parameters:
- name: 'request'
in: body
"""
try:
data = json.loads(to_str(request.data))
lambda_name = data['FunctionName']
event_publisher.fire_event(
event_publisher.EVENT_LAMBDA_CREATE_FUNC,
payload={'n': event_publisher.get_hash(lambda_name)})
arn = func_arn(lambda_name)
if arn in lambda_arn_to_handler:
return error_response('Function already exist: %s' % lambda_name,
409,
error_type='ResourceConflictException')
lambda_arn_to_versions[arn] = {'$LATEST': {'CodeSize': 50}}
lambda_arn_to_handler[arn] = data['Handler']
lambda_arn_to_runtime[arn] = data['Runtime']
lambda_arn_to_envvars[arn] = data.get('Environment',
{}).get('Variables', {})
result = set_function_code(data['Code'], lambda_name)
result.update({
"DeadLetterConfig": data.get('DeadLetterConfig'),
"Description": data.get('Description'),
"Environment": {
"Error": {},
"Variables": lambda_arn_to_envvars[arn]
},
"FunctionArn": arn,
"FunctionName": lambda_name,
"Handler": lambda_arn_to_handler[arn],
"MemorySize": data.get('MemorySize'),
"Role": data.get('Role'),
"Runtime": lambda_arn_to_runtime[arn],
"Timeout": data.get('Timeout'),
"TracingConfig": {},
"VpcConfig": {
"SecurityGroupIds": [None],
"SubnetIds": [None],
"VpcId": None
}
})
return jsonify(result or {})
except Exception as e:
return error_response('Unknown error: %s' % e)
def _fire_event(req_data, response):
action = req_data.get('Action')
event_type = None
queue_url = None
if action == 'CreateQueue':
event_type = event_publisher.EVENT_SQS_CREATE_QUEUE
response_data = xmltodict.parse(response.content)
if 'CreateQueueResponse' in response_data:
queue_url = response_data['CreateQueueResponse']['CreateQueueResult']['QueueUrl']
elif action == 'DeleteQueue':
event_type = event_publisher.EVENT_SQS_DELETE_QUEUE
queue_url = req_data.get('QueueUrl')
if event_type and queue_url:
event_publisher.fire_event(event_type, payload={'u': event_publisher.get_hash(queue_url)})
def stop_infra():
global INFRA_STOPPED
if INFRA_STOPPED:
return
event_publisher.fire_event(event_publisher.EVENT_STOP_INFRA)
generic_proxy.QUIET = True
common.cleanup(files=True, quiet=True)
common.cleanup_resources()
lambda_api.cleanup()
time.sleep(2)
# TODO: optimize this (takes too long currently)
# check_infra(retries=2, expect_shutdown=True)
INFRA_STOPPED = True
def stop_infra():
global INFRA_STOPPED
if INFRA_STOPPED:
return
event_publisher.fire_event(event_publisher.EVENT_STOP_INFRA)
generic_proxy.QUIET = True
common.cleanup(files=True, quiet=True)
common.cleanup_resources()
lambda_api.cleanup()
time.sleep(2)
# TODO: optimize this (takes too long currently)
# check_infra(retries=2, expect_shutdown=True)
INFRA_STOPPED = True
def stop_infra(debug=False):
if common.INFRA_STOPPED:
return
common.INFRA_STOPPED = True
event_publisher.fire_event(event_publisher.EVENT_STOP_INFRA)
generic_proxy.QUIET = True
print_debug('[shutdown] Cleaning up files ...', debug)
common.cleanup(files=True, quiet=True)
print_debug('[shutdown] Cleaning up resources ...', debug)
common.cleanup_resources(debug=debug)
print_debug('[shutdown] Cleaning up Lambda resources ...', debug)
lambda_api.cleanup()
time.sleep(2)
def return_response(self, method, path, data, headers, response):
# persist requests to disk
super(ProxyListenerSNS, self).return_response(
method, path, data, headers, response
)
if method == 'POST' and path == '/':
# convert account IDs in ARNs
data = aws_stack.fix_account_id_in_arns(data, colon_delimiter='%3A')
aws_stack.fix_account_id_in_arns(response)
# remove "None" strings from result
search = r'<entry><key>[^<]+</key>\s*<value>\s*None\s*</[^>]+>\s*</entry>'
response_regex_replace(response, search, '')
# parse request and extract data
req_data = urlparse.parse_qs(to_str(data))
req_action = req_data['Action'][0]
if req_action == 'Subscribe' and response.status_code < 400:
response_data = xmltodict.parse(response.content)
topic_arn = (req_data.get('TargetArn') or req_data.get('TopicArn'))[0]
filter_policy = (req_data.get('FilterPolicy') or [None])[0]
attributes = get_subscribe_attributes(req_data)
sub_arn = response_data['SubscribeResponse']['SubscribeResult']['SubscriptionArn']
do_subscribe(
topic_arn,
req_data['Endpoint'][0],
req_data['Protocol'][0],
sub_arn,
attributes,
filter_policy
)
if req_action == 'CreateTopic' and response.status_code < 400:
response_data = xmltodict.parse(response.content)
topic_arn = response_data['CreateTopicResponse']['CreateTopicResult']['TopicArn']
do_create_topic(topic_arn)
# publish event
event_publisher.fire_event(
event_publisher.EVENT_SNS_CREATE_TOPIC,
payload={'t': event_publisher.get_hash(topic_arn)}
)
if req_action == 'DeleteTopic' and response.status_code < 400:
# publish event
topic_arn = (req_data.get('TargetArn') or req_data.get('TopicArn'))[0]
event_publisher.fire_event(
event_publisher.EVENT_SNS_DELETE_TOPIC,
payload={'t': event_publisher.get_hash(topic_arn)}
)
def return_response(self, method, path, data, headers, response):
# fix backend issue (missing support for API documentation)
if re.match(r'/restapis/[^/]+/documentation/versions', path):
if response.status_code == 404:
return requests_response({'position': '1', 'items': []})
# publish event
if method == 'POST' and path == '/restapis':
content = json.loads(to_str(response.content))
event_publisher.fire_event(event_publisher.EVENT_APIGW_CREATE_API,
payload={'a': event_publisher.get_hash(content['id'])})
api_regex = r'^/restapis/([a-zA-Z0-9\-]+)$'
if method == 'DELETE' and re.match(api_regex, path):
api_id = re.sub(api_regex, r'\1', path)
event_publisher.fire_event(event_publisher.EVENT_APIGW_DELETE_API,
payload={'a': event_publisher.get_hash(api_id)})
def create_domain():
data = json.loads(to_str(request.data))
domain_name = data['DomainName']
if domain_name in ES_DOMAINS:
return error_response(error_type='ResourceAlreadyExistsException')
ES_DOMAINS[domain_name] = data
# start actual Elasticsearch instance
start_elasticsearch_instance()
result = get_domain_status(domain_name)
# record event
event_publisher.fire_event(event_publisher.EVENT_ES_CREATE_DOMAIN,
payload={'n': event_publisher.get_hash(domain_name)})
persistence.record('es', request=request)
return jsonify(result)
def return_response(self, method, path, data, headers, response):
data = json.loads(to_str(data or "{}"))
name = data.get("name") or (data.get("stateMachineArn")
or "").split(":")[-1]
target = headers.get("X-Amz-Target", "").split(".")[-1]
# publish event
if target == "CreateStateMachine":
event_publisher.fire_event(
event_publisher.EVENT_STEPFUNCTIONS_CREATE_SM,
payload={"m": event_publisher.get_hash(name)},
)
elif target == "DeleteStateMachine":
event_publisher.fire_event(
event_publisher.EVENT_STEPFUNCTIONS_DELETE_SM,
payload={"m": event_publisher.get_hash(name)},
)
def return_response(self, method, path, data, headers, response):
# fix backend issue (missing support for API documentation)
if re.match(r"/restapis/[^/]+/documentation/versions", path):
if response.status_code == 404:
return requests_response({"position": "1", "items": []})
# add missing implementations
if response.status_code == 404:
data = data and json.loads(to_str(data))
result = None
if path == "/account":
result = handle_accounts(method, path, data, headers)
elif path.startswith("/vpclinks"):
result = handle_vpc_links(method, path, data, headers)
elif re.match(PATH_REGEX_PATH_MAPPINGS, path):
result = handle_base_path_mappings(method, path, data, headers)
elif re.match(PATH_REGEX_CLIENT_CERTS, path):
result = handle_client_certificates(method, path, data,
headers)
if result is not None:
response.status_code = 200
aws_responses.set_response_content(
response, result, getattr(result, "headers", {}))
# keep track of API regions for faster lookup later on
if method == "POST" and path == "/restapis":
content = json.loads(to_str(response.content))
api_id = content["id"]
region = aws_stack.extract_region_from_auth_header(headers)
API_REGIONS[api_id] = region
# publish event
if method == "POST" and path == "/restapis":
content = json.loads(to_str(response.content))
event_publisher.fire_event(
event_publisher.EVENT_APIGW_CREATE_API,
payload={"a": event_publisher.get_hash(content["id"])},
)
api_regex = r"^/restapis/([a-zA-Z0-9\-]+)$"
if method == "DELETE" and re.match(api_regex, path):
api_id = re.sub(api_regex, r"\1", path)
event_publisher.fire_event(
event_publisher.EVENT_APIGW_DELETE_API,
payload={"a": event_publisher.get_hash(api_id)},
)
def do_start_infra(asynchronous, apis, is_in_docker):
event_publisher.fire_event(event_publisher.EVENT_START_INFRA, {
'd': is_in_docker and 1 or 0,
'c': in_ci() and 1 or 0
})
# set up logging
setup_logging()
# prepare APIs
apis = canonicalize_api_names(apis)
# set environment
os.environ['AWS_REGION'] = config.DEFAULT_REGION
os.environ['ENV'] = ENV_DEV
# register signal handlers
if not os.environ.get(ENV_INTERNAL_TEST_RUN):
register_signal_handlers()
# make sure AWS credentials are configured, otherwise boto3 bails on us
check_aws_credentials()
# install libs if not present
install.install_components(apis)
# Some services take a bit to come up
sleep_time = 5
# start services
thread = None
# loop through plugins and start each service
for name, plugin in SERVICE_PLUGINS.items():
if plugin.is_enabled(api_names=apis):
record_service_health(name, 'starting')
t1 = plugin.start(asynchronous=True)
thread = thread or t1
time.sleep(sleep_time)
# ensure that all infra components are up and running
check_infra(apis=apis)
# restore persisted data
persistence.restore_persisted_data(apis=apis)
print('Ready.')
sys.stdout.flush()
if not asynchronous and thread:
# this is a bit of an ugly hack, but we need to make sure that we
# stay in the execution context of the main thread, otherwise our
# signal handlers don't work
sleep_forever()
return thread
def create_function():
""" Create new function
---
operationId: 'createFunction'
parameters:
- name: 'request'
in: body
"""
arn = 'n/a'
try:
data = json.loads(to_str(request.data))
lambda_name = data['FunctionName']
event_publisher.fire_event(event_publisher.EVENT_LAMBDA_CREATE_FUNC,
payload={'n': event_publisher.get_hash(lambda_name)})
arn = func_arn(lambda_name)
if arn in arn_to_lambda:
return error_response('Function already exist: %s' %
lambda_name, 409, error_type='ResourceConflictException')
arn_to_lambda[arn] = func_details = LambdaFunction(arn)
func_details.versions = {'$LATEST': {'CodeSize': 50}}
func_details.handler = data['Handler']
func_details.runtime = data['Runtime']
func_details.envvars = data.get('Environment', {}).get('Variables', {})
func_details.timeout = data.get('Timeout')
result = set_function_code(data['Code'], lambda_name)
if isinstance(result, Response):
del arn_to_lambda[arn]
return result
result.update({
'DeadLetterConfig': data.get('DeadLetterConfig'),
'Description': data.get('Description'),
'Environment': {'Error': {}, 'Variables': func_details.envvars},
'FunctionArn': arn,
'FunctionName': lambda_name,
'Handler': func_details.handler,
'MemorySize': data.get('MemorySize'),
'Role': data.get('Role'),
'Runtime': func_details.runtime,
'Timeout': data.get('Timeout'),
'TracingConfig': {},
'VpcConfig': {'SecurityGroupIds': [None], 'SubnetIds': [None], 'VpcId': None}
})
return jsonify(result or {})
except Exception as e:
del arn_to_lambda[arn]
return error_response('Unknown error: %s %s' % (e, traceback.format_exc()))
def delete_domain(domain_name):
with _domain_mutex:
if domain_name not in ES_DOMAINS:
return error_response(error_type="ResourceNotFoundException")
result = get_domain_status(domain_name, deleted=True)
del ES_DOMAINS[domain_name]
_cleanup_cluster(domain_name)
# record event
event_publisher.fire_event(
event_publisher.EVENT_ES_DELETE_DOMAIN,
payload={"n": event_publisher.get_hash(domain_name)},
)
persistence.record("es", request=request)
return jsonify(result)
def create_key(
self,
context: RequestContext,
create_key_request: CreateKeyRequest = None,
) -> CreateKeyResponse:
descr = create_key_request.get("Description") or ""
event_publisher.fire_event(EVENT_KMS_CREATE_KEY,
{"k": event_publisher.get_hash(descr)})
result = call_moto(context)
# generate keypair for signing, if this is a SIGN_VERIFY key
key_usage = create_key_request.get("KeyUsage")
if key_usage == "SIGN_VERIFY":
create_key_request["KeyId"] = result["KeyMetadata"]["KeyId"]
_generate_data_key_pair(create_key_request, create_cipher=False)
return result
def _fire_event(req_data, response):
action = req_data.get("Action")
event_type = None
queue_url = None
if action == "CreateQueue":
event_type = event_publisher.EVENT_SQS_CREATE_QUEUE
response_data = xmltodict.parse(response.content)
if "CreateQueueResponse" in response_data:
queue_url = response_data["CreateQueueResponse"][
"CreateQueueResult"]["QueueUrl"]
elif action == "DeleteQueue":
event_type = event_publisher.EVENT_SQS_DELETE_QUEUE
queue_url = req_data.get("QueueUrl")
if event_type and queue_url:
event_publisher.fire_event(
event_type, payload={"u": event_publisher.get_hash(queue_url)})
def create_stream(stream_name,
delivery_stream_type='DirectPut',
delivery_stream_type_configuration=None,
s3_destination=None,
elasticsearch_destination=None,
tags=None,
region_name=None):
tags = tags or {}
stream = {
'DeliveryStreamType': delivery_stream_type,
'KinesisStreamSourceConfiguration': delivery_stream_type_configuration,
'HasMoreDestinations': False,
'VersionId': '1',
'CreateTimestamp': time.time(),
'DeliveryStreamARN': firehose_stream_arn(stream_name),
'DeliveryStreamStatus': 'ACTIVE',
'DeliveryStreamName': stream_name,
'Destinations': [],
'Tags': tags
}
DELIVERY_STREAMS[stream_name] = stream
if elasticsearch_destination:
update_destination(stream_name=stream_name,
destination_id=short_uid(),
elasticsearch_update=elasticsearch_destination)
if s3_destination:
update_destination(stream_name=stream_name,
destination_id=short_uid(),
s3_update=s3_destination)
# record event
event_publisher.fire_event(
event_publisher.EVENT_FIREHOSE_CREATE_STREAM,
payload={'n': event_publisher.get_hash(stream_name)})
if delivery_stream_type == 'KinesisStreamAsSource':
kinesis_stream_name = delivery_stream_type_configuration.get(
'KinesisStreamARN').split('/')[1]
kinesis_connector.listen_to_kinesis(stream_name=kinesis_stream_name,
fh_d_stream=stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix='-firehose',
region_name=region_name)
return stream
def return_response(self, method, path, data, headers, response,
request_handler):
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action', [None])[0]
event_type = None
queue_url = None
if action == 'CreateQueue':
event_type = event_publisher.EVENT_SQS_CREATE_QUEUE
response_data = xmltodict.parse(response.content)
if 'CreateQueueResponse' in response_data:
queue_url = response_data['CreateQueueResponse'][
'CreateQueueResult']['QueueUrl']
elif action == 'DeleteQueue':
event_type = event_publisher.EVENT_SQS_DELETE_QUEUE
queue_url = req_data.get('QueueUrl', [None])[0]
if event_type and queue_url:
event_publisher.fire_event(
event_type,
payload={'u': event_publisher.get_hash(queue_url)})
# patch the response and return the correct endpoint URLs
if action in ('CreateQueue', 'GetQueueUrl', 'ListQueues'):
content_str = content_str_original = to_str(response.content)
new_response = Response()
new_response.status_code = response.status_code
new_response.headers = response.headers
if config.USE_SSL and '<QueueUrl>http://' in content_str:
# return https://... if we're supposed to use SSL
content_str = re.sub(r'<QueueUrl>\s*http://',
r'<QueueUrl>https://', content_str)
# expose external hostname:port
external_port = get_external_port(headers, request_handler)
content_str = re.sub(
r'<QueueUrl>\s*([a-z]+)://[^<]*:([0-9]+)/([^<]*)\s*</QueueUrl>',
r'<QueueUrl>\1://%s:%s/\3</QueueUrl>' %
(HOSTNAME_EXTERNAL, external_port), content_str)
new_response._content = content_str
if content_str_original != new_response._content:
# if changes have been made, return patched response
new_response.headers['content-length'] = len(
new_response._content)
return new_response
def return_response(self, method, path, data, headers, response, request_handler):
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action', [None])[0]
event_type = None
queue_url = None
if action == 'CreateQueue':
event_type = event_publisher.EVENT_SQS_CREATE_QUEUE
response_data = xmltodict.parse(response.content)
if 'CreateQueueResponse' in response_data:
queue_url = response_data['CreateQueueResponse']['CreateQueueResult']['QueueUrl']
elif action == 'DeleteQueue':
event_type = event_publisher.EVENT_SQS_DELETE_QUEUE
queue_url = req_data.get('QueueUrl', [None])[0]
if event_type and queue_url:
event_publisher.fire_event(event_type, payload={'u': event_publisher.get_hash(queue_url)})
# patch the response and return the correct endpoint URLs
if action in ('CreateQueue', 'GetQueueUrl', 'ListQueues'):
content_str = content_str_original = to_str(response.content)
new_response = Response()
new_response.status_code = response.status_code
new_response.headers = response.headers
if config.USE_SSL and '<QueueUrl>http://' in content_str:
# return https://... if we're supposed to use SSL
content_str = re.sub(r'<QueueUrl>\s*http://', r'<QueueUrl>https://', content_str)
# expose external hostname:port
external_port = get_external_port(headers, request_handler)
content_str = re.sub(r'<QueueUrl>\s*([a-z]+)://[^<]*:([0-9]+)/([^<]*)\s*</QueueUrl>',
r'<QueueUrl>\1://%s:%s/\3</QueueUrl>' % (HOSTNAME_EXTERNAL, external_port), content_str)
new_response._content = content_str
if content_str_original != new_response._content:
# if changes have been made, return patched response
new_response.headers['content-length'] = len(new_response._content)
return new_response
# Since the following 2 API calls are not implemented in ElasticMQ, we're mocking them
# and letting them to return an empty response
if action == 'TagQueue':
new_response = Response()
new_response.status_code = 200
new_response._content = (
'<?xml version="1.0"?>'
'<TagQueueResponse>'
'<ResponseMetadata>' # noqa: W291
'<RequestId>{}</RequestId>' # noqa: W291
'</ResponseMetadata>' # noqa: W291
'</TagQueueResponse>'
).format(uuid.uuid4())
return new_response
elif action == 'ListQueueTags':
new_response = Response()
new_response.status_code = 200
new_response._content = (
'<?xml version="1.0"?>'
'<ListQueueTagsResponse xmlns="{}">'
'<ListQueueTagsResult/>' # noqa: W291
'<ResponseMetadata>' # noqa: W291
'<RequestId>{}</RequestId>' # noqa: W291
'</ResponseMetadata>' # noqa: W291
'</ListQueueTagsResponse>'
).format(XMLNS_SQS, uuid.uuid4())
return new_response
time.sleep(2)
t.process.wait()
sys.exit(t.process.returncode)
# -------------
# MAIN STARTUP
# -------------
def start_infra(async=False, apis=None):
try:
# load plugins
load_plugins()
event_publisher.fire_event(event_publisher.EVENT_START_INFRA)
# set up logging
setup_logging()
if not apis:
apis = list(config.SERVICE_PORTS.keys())
# set environment
os.environ['AWS_REGION'] = DEFAULT_REGION
os.environ['ENV'] = ENV_DEV
# register signal handlers
register_signal_handlers()
# make sure AWS credentials are configured, otherwise boto3 bails on us
check_aws_credentials()
# install libs if not present
install.install_components(apis)
def return_response(self, method, path, data, headers, response):
bucket_name = get_bucket_name(path, headers)
# No path-name based bucket name? Try host-based
hostname_parts = headers['host'].split('.')
if (not bucket_name or len(bucket_name) == 0) and len(hostname_parts) > 1:
bucket_name = hostname_parts[0]
# POST requests to S3 may include a success_action_redirect field,
# which should be used to redirect a client to a new location.
key = None
if method == 'POST':
key, redirect_url = multipart_content.find_multipart_redirect_url(data, headers)
if key and redirect_url:
response.status_code = 303
response.headers['Location'] = expand_redirect_url(redirect_url, key, bucket_name)
LOGGER.debug('S3 POST {} to {}'.format(response.status_code, response.headers['Location']))
parsed = urlparse.urlparse(path)
bucket_name_in_host = headers['host'].startswith(bucket_name)
should_send_notifications = all([
method in ('PUT', 'POST', 'DELETE'),
'/' in path[1:] or bucket_name_in_host,
# check if this is an actual put object request, because it could also be
# a put bucket request with a path like this: /bucket_name/
bucket_name_in_host or (len(path[1:].split('/')) > 1 and len(path[1:].split('/')[1]) > 0),
# don't send notification if url has a query part (some/path/with?query)
# (query can be one of 'notification', 'lifecycle', 'tagging', etc)
not parsed.query
])
# get subscribers and send bucket notifications
if should_send_notifications:
# if we already have a good key, use it, otherwise examine the path
if key:
object_path = '/' + key
elif bucket_name_in_host:
object_path = parsed.path
else:
parts = parsed.path[1:].split('/', 1)
object_path = parts[1] if parts[1][0] == '/' else '/%s' % parts[1]
send_notifications(method, bucket_name, object_path)
# publish event for creation/deletion of buckets:
if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0):
event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT'
else event_publisher.EVENT_S3_DELETE_BUCKET)
event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(bucket_name)})
# fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382)
if method == 'PUT' and parsed.query == 'policy':
response._content = ''
response.status_code = 204
return response
if response:
# append CORS headers to response
append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response)
response_content_str = None
try:
response_content_str = to_str(response._content)
except Exception:
pass
# we need to un-pretty-print the XML, otherwise we run into this issue with Spark:
# https://github.com/jserver/mock-s3/pull/9/files
# https://github.com/localstack/localstack/issues/183
# Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n
if response_content_str and response_content_str.startswith('<'):
is_bytes = isinstance(response._content, six.binary_type)
response._content = re.sub(r'([^\?])>\n\s*<', r'\1><', response_content_str, flags=re.MULTILINE)
if is_bytes:
response._content = to_bytes(response._content)
# fix content-type: https://github.com/localstack/localstack/issues/618
# https://github.com/localstack/localstack/issues/549
if 'text/html' in response.headers.get('Content-Type', ''):
response.headers['Content-Type'] = 'application/xml; charset=utf-8'
response.headers['content-length'] = len(response._content)
# update content-length headers (fix https://github.com/localstack/localstack/issues/541)
if method == 'DELETE':
response.headers['content-length'] = len(response._content)
def return_response(self, method, path, data, headers, response):
data = json.loads(to_str(data))
# update table definitions
if data and 'TableName' in data and 'KeySchema' in data:
TABLE_DEFINITIONS[data['TableName']] = data
if response._content:
# fix the table ARN (DynamoDBLocal hardcodes "ddblocal" as the region)
content_replaced = re.sub(r'"TableArn"\s*:\s*"arn:aws:dynamodb:ddblocal:([^"]+)"',
r'"TableArn": "arn:aws:dynamodb:%s:\1"' % aws_stack.get_local_region(), to_str(response._content))
if content_replaced != response._content:
response._content = content_replaced
fix_headers_for_updated_response(response)
action = headers.get('X-Amz-Target')
if not action:
return
record = {
'eventID': '1',
'eventVersion': '1.0',
'dynamodb': {
'StreamViewType': 'NEW_AND_OLD_IMAGES',
'SizeBytes': -1
},
'awsRegion': DEFAULT_REGION,
'eventSource': 'aws:dynamodb'
}
records = [record]
if action == '%s.UpdateItem' % ACTION_PREFIX:
updated_item = find_existing_item(data)
if not updated_item:
return
record['eventName'] = 'MODIFY'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb']['OldImage'] = ProxyListenerDynamoDB.thread_local.existing_item
record['dynamodb']['NewImage'] = updated_item
record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item))
elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
records = []
for table_name, requests in data['RequestItems'].items():
for request in requests:
put_request = request.get('PutRequest')
if put_request:
keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name)
if isinstance(keys, Response):
return keys
new_record = clone(record)
new_record['eventName'] = 'INSERT'
new_record['dynamodb']['Keys'] = keys
new_record['dynamodb']['NewImage'] = put_request['Item']
new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
records.append(new_record)
elif action == '%s.PutItem' % ACTION_PREFIX:
existing_item = ProxyListenerDynamoDB.thread_local.existing_item
ProxyListenerDynamoDB.thread_local.existing_item = None
record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
keys = dynamodb_extract_keys(item=data['Item'], table_name=data['TableName'])
if isinstance(keys, Response):
return keys
record['dynamodb']['Keys'] = keys
record['dynamodb']['NewImage'] = data['Item']
record['dynamodb']['SizeBytes'] = len(json.dumps(data['Item']))
elif action == '%s.GetItem' % ACTION_PREFIX:
if response.status_code == 200:
content = json.loads(to_str(response.content))
# make sure we append 'ConsumedCapacity', which is properly
# returned by dynalite, but not by AWS's DynamoDBLocal
if 'ConsumedCapacity' not in content and data.get('ReturnConsumedCapacity') in ('TOTAL', 'INDEXES'):
content['ConsumedCapacity'] = {
'CapacityUnits': 0.5, # TODO hardcoded
'TableName': data['TableName']
}
response._content = json.dumps(content)
fix_headers_for_updated_response(response)
elif action == '%s.DeleteItem' % ACTION_PREFIX:
old_item = ProxyListenerDynamoDB.thread_local.existing_item
record['eventName'] = 'REMOVE'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb']['OldImage'] = old_item
elif action == '%s.CreateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
event_publisher.fire_event(event_publisher.EVENT_DYNAMODB_CREATE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.DeleteTable' % ACTION_PREFIX:
event_publisher.fire_event(event_publisher.EVENT_DYNAMODB_DELETE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.UpdateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
return
else:
# nothing to do
return
if len(records) > 0 and 'eventName' in records[0]:
if 'TableName' in data:
records[0]['eventSourceARN'] = aws_stack.dynamodb_table_arn(data['TableName'])
forward_to_lambda(records)
forward_to_ddb_stream(records)
