def add_unparsed_request(parse_params: HttpRequest):
"""
This function handle the case where we got a request the is not fully formatted as we expected,
I.e. there isn't '\r\n' in the request data that <i>logically</i> splits the headers from the body.
In that case, we will consider it as a continuance of the previous request if they got the same url,
and we didn't get any answer yet.
"""
if is_lumigo_edge(parse_params.host):
return
last_event = SpansContainer.get_span().get_last_span()
if last_event:
if last_event and last_event.get(
"type") == HTTP_TYPE and HttpState.previous_request:
if last_event.get("info",
{}).get("httpInfo",
{}).get("host") == parse_params.host:
if "response" not in last_event["info"]["httpInfo"]:
SpansContainer.get_span().pop_last_span()
body = (HttpState.previous_request.body +
parse_params.body)[:get_size_upper_bound()]
add_request_event(
HttpState.previous_request.clone(body=body))
return
add_request_event(parse_params.clone(headers=None))
def test_lambda_parser_resource_name(uri, resource_name):
parser = LambdaParser()
params = HttpRequest(
host="", method="POST", uri=uri, headers={}, body=json.dumps({"hello": "world"})
)
response = parser.parse_request(params)
assert response["info"]["resourceName"] == resource_name
def test_s3_parser_resource_name(uri, resource_name, host):
parser = S3Parser()
params = HttpRequest(
host=host,
method="PUT",
uri=uri,
headers={},
body="",
)
response = parser.parse_request(params)
assert response["info"]["resourceName"] == resource_name
def test_dynamodb_parser_sad_flow_unsupported_query():
parser = DynamoParser()
params = HttpRequest(
host="",
method="POST",
uri="",
headers={"x-amz-target": "DynamoDB_20120810.BatchWriteItem"},
body='{"RequestItems": {}}',
)
with pytest.raises(Exception):
parser.parse_request(params)
def test_dynamodb_parse_no_scrubbing():
body = {"TableName": "component-test", "Key": {"field0": {"S": "1"}}}
parser = DynamoParser()
params = HttpRequest(
host="",
method="POST",
uri="",
headers={"x-amz-target": "DynamoDB_20120810.GetItem"},
body=json.dumps(body),
)
response = parser.parse_request(params)
assert json.loads(response["info"]["httpInfo"]["request"]["body"]) == body
def test_dynamodb_parser_sad_flow():
parser = DynamoParser()
params = HttpRequest(
host="",
method="POST",
uri="",
headers={"x-amz-target": "DynamoDB_20120810.GetItem"},
body="not a json",
)
response = parser.parse_request(params)
assert response["info"]["resourceName"] is None
assert response["info"]["dynamodbMethod"] == "GetItem"
assert response["info"]["messageId"] is None
def test_dynamodb_parser_happy_flow(method, body, message_id):
parser = DynamoParser()
params = HttpRequest(
host="",
method="POST",
uri="",
headers={"x-amz-target": f"DynamoDB_20120810.{method}"},
body=json.dumps(body),
)
response = parser.parse_request(params)
assert response["info"]["resourceName"] == "resourceName"
assert response["info"]["dynamodbMethod"] == method
assert response["info"]["messageId"] == message_id
async def on_request_start(session, trace_config_ctx, params):
with lumigo_safe_execute("aiohttp on_request_start"):
span = add_request_event(
span_id=None,
parse_params=HttpRequest(
host=params.url.host,
method=params.method,
uri=str(params.url),
headers=dict(params.headers),
body=b"",
),
)
setattr(trace_config_ctx, LUMIGO_SPAN_ID_KEY, span["id"])
def test_aggregating_response_body():
"""
This test is here to validate that we're not leaking memory on aggregating response body.
Unfortunately python doesn't give us better tools, so we must check the problematic member itself.
"""
SpansContainer.create_span()
add_request_event(
HttpRequest(host="dummy",
method="dummy",
uri="dummy",
headers={"dummy": "dummy"},
body="dummy"))
big_response_chunk = b"leak" * DEFAULT_MAX_ENTRY_SIZE
for _ in range(10):
update_event_response(host=None,
status_code=200,
headers=None,
body=big_response_chunk)
assert len(HttpState.previous_response_body) <= len(big_response_chunk)
def _requests_wrapper(func, instance, args, kwargs):
"""
This is the wrapper of the function `requests.request`.
This function is being wrapped specifically because it initializes the connection by itself and parses the response,
which creates a gap from the traditional http.client wrapping.
Moreover, these "extra" steps may raise exceptions. We should attach the error to the http span.
"""
start_time = datetime.now()
try:
ret_val = func(*args, **kwargs)
except Exception as exception:
with lumigo_safe_execute("requests wrapper exception occurred"):
method = safe_get_list(args, 0, kwargs.get("method", "")).upper()
url = safe_get_list(args, 1, kwargs.get("url"))
if Configuration.is_sync_tracer:
if HttpState.previous_request:
span = SpansContainer.get_span().get_span_by_id(
HttpState.previous_span_id)
else:
span = add_request_event(
None,
HttpRequest(
host=url,
method=method,
uri=url,
body=kwargs.get("data"),
headers=kwargs.get("headers"),
instance_id=id(instance),
),
)
span_id = span["id"]
HttpState.request_id_to_span_id[get_lumigo_connection_id(
instance)] = span_id
SpansContainer.add_exception_to_span(span, exception, [])
raise
with lumigo_safe_execute("requests wrapper time updates"):
span_id = HttpState.response_id_to_span_id.get(
get_lumigo_connection_id(ret_val.raw._original_response))
SpansContainer.get_span().update_event_times(span_id,
start_time=start_time)
return ret_val
def test_aggregating_response_body():
"""
This test is here to validate that we're not leaking memory on aggregating response body.
Unfortunately python doesn't give us better tools, so we must check the problematic member itself.
"""
SpansContainer.create_span()
span = add_request_event(
None,
HttpRequest(
host="dummy", method="dummy", uri="dummy", headers={"dummy": "dummy"}, body="dummy"
),
)
big_response_chunk = b'leak"' * DEFAULT_MAX_ENTRY_SIZE
for _ in range(10):
update_event_response(
span["id"], host=None, status_code=200, headers=None, body=big_response_chunk
)
body = list(SpansContainer.get_span().spans.values())[0]["info"]["httpInfo"]["response"]["body"]
assert len(body) <= len(big_response_chunk)
assert body[: -len(TRUNCATE_SUFFIX)] in json.dumps(big_response_chunk.decode())
def test_event_bridge_parser_request_happy_flow():
parser = EventBridgeParser()
params = HttpRequest(
host="",
method="POST",
uri="",
headers={},
body=json.dumps(
{
"Entries": [
{
"Source": "source_lambda",
"Resources": [],
"DetailType": "string",
"Detail": '{"a": 1}',
"EventBusName": "name1",
},
{
"Source": "source_lambda",
"Resources": [],
"DetailType": "string",
"Detail": '{"a": 2}',
"EventBusName": "name1",
},
{
"Source": "source_lambda",
"Resources": [],
"DetailType": "string",
"Detail": '{"a": 3}',
"EventBusName": "name2",
},
]
}
),
)
response = parser.parse_request(params)
assert set(response["info"]["resourceNames"]) == {"name2", "name1"}
def _http_send_wrapper(func, instance, args, kwargs):
"""
This is the wrapper of the requests. it parses the http's message to conclude the url, headers, and body.
Finally, it add an event to the span, and run the wrapped function (http.client.HTTPConnection.send).
"""
data = safe_get_list(args, 0)
with lumigo_safe_execute("parse requested streams"):
if hasattr(data, "read"):
if not hasattr(data, "seek") or not hasattr(data, "tell"):
# If we will read this data, then we will change the original behavior
data = ""
else:
current_pos = data.tell()
data = data.read(get_size_upper_bound())
args[0].seek(current_pos)
host, method, headers, body, uri = (
getattr(instance, "host", None),
getattr(instance, "_method", None),
None,
None,
None,
)
with lumigo_safe_execute("parse request"):
if isinstance(data, bytes) and _BODY_HEADER_SPLITTER in data:
headers, body = data.split(_BODY_HEADER_SPLITTER, 1)
hooked_headers = getattr(instance, LUMIGO_HEADERS_HOOK_KEY, None)
if hooked_headers and hooked_headers.headers:
# we will get here only if _headers_reminder_wrapper ran first. remove its traces.
headers = {
k: ensure_str(v)
for k, v in hooked_headers.headers.items()
}
uri = f"{host}{hooked_headers.path}"
setattr(instance, LUMIGO_HEADERS_HOOK_KEY, None)
elif _FLAGS_HEADER_SPLITTER in headers:
request_info, headers = headers.split(_FLAGS_HEADER_SPLITTER,
1)
headers = http.client.parse_headers(BytesIO(headers))
path_and_query_params = (
# Parse path from request info, remove method (GET | POST) and http version (HTTP/1.1)
request_info.decode("ascii").replace(method, "").replace(
instance._http_vsn_str, "").strip())
uri = f"{host}{path_and_query_params}"
host = host or headers.get("Host")
else:
headers = None
with lumigo_safe_execute("add request event"):
if headers:
add_request_event(
HttpRequest(host=host,
method=method,
uri=uri,
headers=headers,
body=body))
else:
add_unparsed_request(
HttpRequest(host=host, method=method, uri=uri, body=data))
ret_val = func(*args, **kwargs)
with lumigo_safe_execute("add response event"):
SpansContainer.get_span().update_event_end_time()
return ret_val
def test_event_bridge_parser_request_sad_flow():
parser = EventBridgeParser()
params = HttpRequest(host="", method="POST", uri="", headers={}, body="not a json")
response = parser.parse_request(params)
assert response["info"]["resourceNames"] is None