def forward_request(self, method, path, data, headers):
if method == 'OPTIONS':
return 200
if path.split('?')[0] == '/health':
return serve_health_endpoint(method, path, data)
if method == 'POST' and path == '/graph':
return serve_resource_graph(data)
# kill the process if we receive this header
headers.get(HEADER_KILL_SIGNAL) and os._exit(0)
target = headers.get('x-amz-target', '')
auth_header = headers.get('authorization', '')
host = headers.get('host', '')
headers[HEADER_LOCALSTACK_EDGE_URL] = 'https://%s' % host
# extract API details
api, port, path, host = get_api_from_headers(headers, path)
set_default_region_in_headers(headers)
if port and int(port) < 0:
return 404
if not port:
port = get_port_from_custom_rules(method, path, data,
headers) or port
if not port:
if api in ['', None, '_unknown_']:
truncated = truncate(data)
LOG.info((
'Unable to find forwarding rule for host "%s", path "%s", '
'target header "%s", auth header "%s", data "%s"') %
(host, path, target, auth_header, truncated))
else:
LOG.info((
'Unable to determine forwarding port for API "%s" - please '
'make sure this API is enabled via the SERVICES configuration'
) % api)
response = Response()
response.status_code = 404
response._content = '{"status": "running"}'
return response
connect_host = '%s:%s' % (config.HOSTNAME, port)
url = '%s://%s%s' % (get_service_protocol(), connect_host, path)
headers['Host'] = host
function = getattr(requests, method.lower())
if isinstance(data, dict):
data = json.dumps(data)
response = function(url,
data=data,
headers=headers,
verify=False,
stream=True)
return response
def handle_connection(self, conn):
socket_file = conn.makefile()
while self.running:
line = socket_file.readline()
line = line[:-1]
if line == '':
# end of socket input stream
break
else:
try:
event = json.loads(line)
records = event['records']
shard_id = event['shard_id']
method_args = inspect.getargspec(self.callback)[0]
if len(method_args) > 2:
self.callback(records,
shard_id=shard_id,
fh_d_stream=self.fh_d_stream)
elif len(method_args) > 1:
self.callback(records, shard_id=shard_id)
else:
self.callback(records)
except Exception as e:
LOGGER.warning(
"Unable to process JSON line: '%s': %s %s. Callback: %s"
% (truncate(line), e, traceback.format_exc(),
self.callback))
conn.close()
def _put_to_search_db(
self, db_flavor, db_description, delivery_stream_name, records, unprocessed_records
):
"""
sends Firehose records to an ElasticSearch or Opensearch database
"""
search_db_index = db_description["IndexName"]
search_db_type = db_description.get("TypeName")
region = aws_stack.get_region()
domain_arn = db_description.get("DomainARN")
cluster_endpoint = db_description.get("ClusterEndpoint")
if cluster_endpoint is None:
cluster_endpoint = aws_stack.get_opensearch_endpoint(domain_arn)
db_connection = get_search_db_connection(cluster_endpoint, region)
if db_description.get("S3BackupMode") == ElasticsearchS3BackupMode.AllDocuments:
s3_dest_desc = db_description.get("S3DestinationDescription")
if s3_dest_desc:
try:
self._put_records_to_s3_bucket(
stream_name=delivery_stream_name,
records=unprocessed_records,
s3_destination_description=s3_dest_desc,
)
except Exception as e:
LOG.warning("Unable to backup unprocessed records to S3. Error: %s", e)
else:
LOG.warning("Passed S3BackupMode without S3Configuration. Cannot backup...")
elif db_description.get("S3BackupMode") == ElasticsearchS3BackupMode.FailedDocumentsOnly:
# TODO support FailedDocumentsOnly as well
LOG.warning("S3BackupMode FailedDocumentsOnly is set but currently not supported.")
for record in records:
obj_id = uuid.uuid4()
data = "{}"
# DirectPut
if "Data" in record:
data = base64.b64decode(record["Data"])
# KinesisAsSource
elif "data" in record:
data = base64.b64decode(record["data"])
try:
body = json.loads(data)
except Exception as e:
LOG.warning(f"{db_flavor} only allows json input data!")
raise e
LOG.debug(
"Publishing to {} destination. Data: {}".format(
db_flavor, truncate(data, max_length=300)
)
)
try:
db_connection.create(
index=search_db_index, doc_type=search_db_type, id=obj_id, body=body
)
except Exception as e:
LOG.exception(f"Unable to put record to stream {delivery_stream_name}.")
raise e
def raise_exception_if_error_response(response):
if not is_response_obj(response):
return
if response.status_code < 400:
return
content = "..."
try:
content = truncate(to_str(response.content or ""))
except Exception:
pass # ignore if content has non-printable bytes
raise Exception("Received error response (code %s): %s" % (response.status_code, content))
def forward_request(self, method, path, data, headers):
if path.split('?')[0] == '/health':
return serve_health_endpoint(method, path, data)
if method == 'POST' and path == '/graph':
return serve_resource_graph(data)
# kill the process if we receive this header
headers.get(HEADER_KILL_SIGNAL) and os._exit(0)
target = headers.get('x-amz-target', '')
auth_header = headers.get('authorization', '')
host = headers.get('host', '')
headers[HEADER_LOCALSTACK_EDGE_URL] = 'https://%s' % host
# extract API details
api, port, path, host = get_api_from_headers(headers, method=method, path=path, data=data)
set_default_region_in_headers(headers)
if port and int(port) < 0:
return 404
if not port:
api, port = get_api_from_custom_rules(method, path, data, headers) or (api, port)
if not port:
if method == 'OPTIONS':
return 200
if api in ['', None, '_unknown_']:
truncated = truncate(data)
LOG.info(('Unable to find forwarding rule for host "%s", path "%s %s", '
'target header "%s", auth header "%s", data "%s"') % (
host, method, path, target, auth_header, truncated))
else:
LOG.info(('Unable to determine forwarding port for API "%s" - please '
'make sure this API is enabled via the SERVICES configuration') % api)
response = Response()
response.status_code = 404
response._content = '{"status": "running"}'
return response
if api and not headers.get('Authorization'):
headers['Authorization'] = aws_stack.mock_aws_request_headers(api)['Authorization']
headers['Host'] = host
if isinstance(data, dict):
data = json.dumps(data)
return do_forward_request(api, port, method, path, data, headers)
def test_truncate(self):
env = common.truncate('foobar', 3)
self.assertEqual(env, 'foo...')
def forward_request(self, method, path, data, headers):
if path.split('?')[0] == '/health':
return serve_health_endpoint(method, path, data)
if method == 'POST' and path == '/graph':
return serve_resource_graph(data)
# kill the process if we receive this header
headers.get(HEADER_KILL_SIGNAL) and os._exit(0)
target = headers.get('x-amz-target', '')
auth_header = headers.get('authorization', '')
host = headers.get('host', '')
headers[HEADER_LOCALSTACK_EDGE_URL] = 'https://%s' % host
# extract API details
api, port, path, host = get_api_from_headers(headers, method=method, path=path, data=data)
set_default_region_in_headers(headers)
if port and int(port) < 0:
return 404
if not port:
api, port = get_api_from_custom_rules(method, path, data, headers) or (api, port)
if not port:
if method == 'OPTIONS':
return 200
if api in ['', None, '_unknown_']:
truncated = truncate(data)
if auth_header or target or data or path not in ['/', '/favicon.ico']:
LOG.info(('Unable to find forwarding rule for host "%s", path "%s %s", '
'target header "%s", auth header "%s", data "%s"') % (
host, method, path, target, auth_header, truncated))
else:
LOG.info(('Unable to determine forwarding port for API "%s" - please '
'make sure this API is enabled via the SERVICES configuration') % api)
response = Response()
response.status_code = 404
response._content = '{"status": "running"}'
return response
if api and not headers.get('Authorization'):
headers['Authorization'] = aws_stack.mock_aws_request_headers(api)['Authorization']
headers['Host'] = host
if isinstance(data, dict):
data = json.dumps(data)
encoding_type = headers.get('content-encoding') or ''
if encoding_type.upper() == GZIP_ENCODING and api is not S3:
headers.set('content-encoding', IDENTITY_ENCODING)
data = gzip.decompress(data)
lock_ctx = BOOTSTRAP_LOCK
if persistence.API_CALLS_RESTORED or is_internal_call_context(headers):
lock_ctx = empty_context_manager()
with lock_ctx:
return do_forward_request(api, method, path, data, headers, port=port)
def test_truncate(self):
env = common.truncate("foobar", 3)
assert env == "foo..."
def forward_request(self, method, path, data, headers):
if common.INFRA_STOPPED:
return 503
if config.EDGE_FORWARD_URL:
return do_forward_request_network(
0,
method,
path,
data,
headers,
target_url=config.EDGE_FORWARD_URL)
# kill the process if we receive this header
headers.get(HEADER_KILL_SIGNAL) and sys.exit(0)
target = headers.get("x-amz-target", "")
auth_header = get_auth_string(method, path, headers, data)
if auth_header and not headers.get("authorization"):
headers["authorization"] = auth_header
host = headers.get("host", "")
orig_req_url = headers.pop(HEADER_LOCALSTACK_REQUEST_URL, "")
headers[HEADER_LOCALSTACK_EDGE_URL] = (re.sub(
r"^([^:]+://[^/]+).*", r"\1", orig_req_url) or "http://%s" % host)
# extract API details
api, port, path, host = get_api_from_headers(headers,
method=method,
path=path,
data=data)
set_default_region_in_headers(headers)
if port and int(port) < 0:
return 404
if not port:
api, port = get_api_from_custom_rules(method, path, data,
headers) or (
api,
port,
)
should_log_trace = is_trace_logging_enabled(headers)
if api and should_log_trace:
# print request trace for debugging, if enabled
LOG.debug('IN(%s): "%s %s" - headers: %s - data: %s', api, method,
path, dict(headers), data)
if not port:
if method == "OPTIONS":
if api and should_log_trace:
# print request trace for debugging, if enabled
LOG.debug('IN(%s): "%s %s" - status: %s', api, method,
path, 200)
return 200
if api in ["", None, API_UNKNOWN]:
truncated = truncate(data)
if auth_header or target or data or path not in [
"/", "/favicon.ico"
]:
LOG.info(
('Unable to find forwarding rule for host "%s", path "%s %s", '
'target header "%s", auth header "%s", data "%s"'),
host,
method,
path,
target,
auth_header,
truncated,
)
else:
LOG.info(
('Unable to determine forwarding port for API "%s" - please '
"make sure this API is enabled via the SERVICES configuration"
),
api,
)
response = Response()
response.status_code = 404
response._content = '{"status": "running"}'
return response
if api and not headers.get("Authorization"):
headers["Authorization"] = aws_stack.mock_aws_request_headers(
api)["Authorization"]
headers[HEADER_TARGET_API] = str(api)
headers["Host"] = host
if isinstance(data, dict):
data = json.dumps(data)
encoding_type = headers.get("Content-Encoding") or ""
if encoding_type.upper() == GZIP_ENCODING.upper(
) and api not in SKIP_GZIP_APIS:
headers.set("Content-Encoding", IDENTITY_ENCODING)
data = gzip.decompress(data)
is_internal_call = is_internal_call_context(headers)
self._require_service(api)
lock_ctx = BOOTSTRAP_LOCK
if is_internal_call or persistence.is_persistence_restored():
lock_ctx = empty_context_manager()
with lock_ctx:
result = do_forward_request(api,
method,
path,
data,
headers,
port=port)
if should_log_trace and result not in [None, False, True]:
result_status_code = getattr(result, "status_code", result)
result_headers = getattr(result, "headers", {})
result_content = getattr(result, "content", "")
LOG.debug(
'OUT(%s): "%s %s" - status: %s - response headers: %s - response: %s',
api,
method,
path,
result_status_code,
dict(result_headers or {}),
result_content,
)
return result
def test_truncate(self):
env = common.truncate("foobar", 3)
self.assertEqual("foo...", env)
def forward_request(self, method, path, data, headers):
if path.split('?')[0] == '/health':
return serve_health_endpoint(method, path, data)
if method == 'POST' and path == '/graph':
return serve_resource_graph(data)
# kill the process if we receive this header
headers.get(HEADER_KILL_SIGNAL) and sys.exit(0)
target = headers.get('x-amz-target', '')
auth_header = get_auth_string(method, path, headers, data)
if auth_header and not headers.get('authorization'):
headers['authorization'] = auth_header
host = headers.get('host', '')
orig_req_url = headers.pop(HEADER_LOCALSTACK_REQUEST_URL, '')
headers[HEADER_LOCALSTACK_EDGE_URL] = (re.sub(
r'^([^:]+://[^/]+).*', r'\1', orig_req_url) or 'http://%s' % host)
# extract API details
api, port, path, host = get_api_from_headers(headers,
method=method,
path=path,
data=data)
if api and config.LS_LOG:
# print request trace for debugging, if enabled
LOG.debug('IN(%s): "%s %s" - headers: %s - data: %s' %
(api, method, path, dict(headers), data))
set_default_region_in_headers(headers)
if port and int(port) < 0:
return 404
if not port:
api, port = get_api_from_custom_rules(method, path, data,
headers) or (api, port)
if not port:
if method == 'OPTIONS':
if api and config.LS_LOG:
# print request trace for debugging, if enabled
LOG.debug('OUT(%s): "%s %s" - status: %s' %
(api, method, path, 200))
return 200
if api in ['', None, API_UNKNOWN]:
truncated = truncate(data)
if auth_header or target or data or path not in [
'/', '/favicon.ico'
]:
LOG.info((
'Unable to find forwarding rule for host "%s", path "%s %s", '
'target header "%s", auth header "%s", data "%s"') %
(host, method, path, target, auth_header,
truncated))
else:
LOG.info((
'Unable to determine forwarding port for API "%s" - please '
'make sure this API is enabled via the SERVICES configuration'
) % api)
response = Response()
response.status_code = 404
response._content = '{"status": "running"}'
return response
if api and not headers.get('Authorization'):
headers['Authorization'] = aws_stack.mock_aws_request_headers(
api)['Authorization']
headers[HEADER_TARGET_API] = str(api)
headers['Host'] = host
if isinstance(data, dict):
data = json.dumps(data)
encoding_type = headers.get('Content-Encoding') or ''
if encoding_type.upper() == GZIP_ENCODING.upper() and api not in [S3]:
headers.set('Content-Encoding', IDENTITY_ENCODING)
data = gzip.decompress(data)
lock_ctx = BOOTSTRAP_LOCK
if persistence.API_CALLS_RESTORED or is_internal_call_context(headers):
lock_ctx = empty_context_manager()
with lock_ctx:
return do_forward_request(api,
method,
path,
data,
headers,
port=port)
