def get_data_from_request(request, capture_body=False, capture_headers=True):
result = {
"env": dict(get_environ(request.env)),
"method": request.method,
"socket": {"remote_address": request.env.get("REMOTE_ADDR"),
"encrypted": True if request.scheme == 'https' else False},
"cookies": request.cookies,
}
if capture_headers:
result["headers"] = dict(get_headers(request.env))
if request.method in constants.HTTP_WITH_BODY:
body = None
if request.content_type == "application/x-www-form-urlencoded":
body = compat.multidict_to_dict(request.stream.read())
elif request.content_type and request.content_type.startswith("multipart/form-data"):
body = compat.multidict_to_dict(request.stream.read())
else:
try:
body = request.stream.read()
except ClientDisconnected:
pass
if body is not None:
result["body"] = body if capture_body else "[REDACTED]"
result["url"] = get_url_dict(request.url)
return result
def get_data_from_request(request, capture_body=False):
result = {
'env': dict(get_environ(request.environ)),
'headers': dict(
get_headers(request.environ),
),
'method': request.method,
'socket': {
'remote_address': request.environ.get('REMOTE_ADDR'),
'encrypted': request.is_secure
},
'cookies': request.cookies,
}
if request.method not in ('GET', 'HEAD'):
body = None
if request.content_type == 'application/x-www-form-urlencoded':
body = compat.multidict_to_dict(request.form)
elif request.content_type.startswith('multipart/form-data'):
body = compat.multidict_to_dict(request.form)
if request.files:
body['_files'] = {
field: val[0].filename if len(val) == 1 else [f.filename for f in val]
for field, val in compat.iterlists(request.files)
}
else:
try:
body = request.data
except ClientDisconnected:
pass
if body is not None:
result['body'] = body if capture_body else '[REDACTED]'
result['url'] = get_url_dict(request.url)
return result
def get_data_from_request(request, config, event_type):
result = {
"env": dict(get_environ(request.environ)),
"method": request.method,
"socket": {"remote_address": request.environ.get("REMOTE_ADDR")},
"cookies": request.cookies,
}
if config.capture_headers:
result["headers"] = dict(get_headers(request.environ))
if request.method in constants.HTTP_WITH_BODY:
if config.capture_body not in ("all", event_type):
result["body"] = "[REDACTED]"
else:
body = None
if request.content_type == "application/x-www-form-urlencoded":
body = compat.multidict_to_dict(request.form)
elif request.content_type and request.content_type.startswith("multipart/form-data"):
body = compat.multidict_to_dict(request.form)
if request.files:
body["_files"] = {
field: val[0].filename if len(val) == 1 else [f.filename for f in val]
for field, val in compat.iterlists(request.files)
}
else:
try:
body = request.get_data(as_text=True)
except ClientDisconnected:
pass
if body is not None:
result["body"] = body
result["url"] = get_url_dict(request.url)
return result
async def get_data_from_request(request: Request, config: Config, event_type: str) -> dict:
"""Loads data from incoming request for APM capturing.
Args:
request (Request)
config (Config)
event_type (str)
Returns:
dict
"""
result = {
"method": request.method,
"socket": {"remote_address": _get_client_ip(request)},
"cookies": request.cookies,
}
if config.capture_headers:
result["headers"] = dict(request.headers)
if request.method in constants.HTTP_WITH_BODY:
if config.capture_body not in ("all", event_type):
result["body"] = "[REDACTED]"
else:
body = None
try:
body = await get_body(request)
except Exception:
pass
if body is not None:
result["body"] = body
result["url"] = get_url_dict(str(request.url))
return result
def get_data_from_request(self, request, event_type):
result = {
"env": dict(get_environ(request.META)),
"method": request.method,
"socket": {
"remote_address": request.META.get("REMOTE_ADDR"),
"encrypted": request.is_secure()
},
"cookies": dict(request.COOKIES),
}
if self.config.capture_headers:
request_headers = dict(get_headers(request.META))
for key, value in request_headers.items():
if isinstance(value, (int, float)):
request_headers[key] = str(value)
result["headers"] = request_headers
if request.method in constants.HTTP_WITH_BODY:
content_type = request.META.get("CONTENT_TYPE")
if content_type == "application/x-www-form-urlencoded":
data = compat.multidict_to_dict(request.POST)
elif content_type and content_type.startswith(
"multipart/form-data"):
data = compat.multidict_to_dict(request.POST)
if request.FILES:
data["_files"] = {
field: file.name
for field, file in compat.iteritems(request.FILES)
}
else:
try:
data = request.body
except Exception as e:
self.logger.debug("Can't capture request body: %s",
compat.text_type(e))
data = "<unavailable>"
capture_body = self.config.capture_body in ("all", event_type)
result["body"] = data if (capture_body
or not data) else "[REDACTED]"
if hasattr(request, "get_raw_uri"):
# added in Django 1.9
url = request.get_raw_uri()
else:
try:
# Requires host to be in ALLOWED_HOSTS, might throw a
# DisallowedHost exception
url = request.build_absolute_uri()
except DisallowedHost:
# We can't figure out the real URL, so we have to set it to
# DisallowedHost
result["url"] = {"full": "DisallowedHost"}
url = None
if url:
result["url"] = get_url_dict(url)
return result
def test_get_url_dict():
data = {
"http://example.com": {
"protocol": "http:",
"hostname": "example.com",
"pathname": "",
"full": "http://example.com",
},
"http://example.com:443": {
"protocol": "http:",
"hostname": "example.com",
"port": "443",
"pathname": "",
"full": "http://example.com:443",
},
"http://example.com:443/a/b/c": {
"protocol": "http:",
"hostname": "example.com",
"port": "443",
"pathname": "/a/b/c",
"full": "http://example.com:443/a/b/c",
},
"https://example.com:443/": {
"protocol": "https:",
"hostname": "example.com",
"port": "443",
"pathname": "/",
"full": "https://example.com:443/",
},
"https://example.com:443/a/b/c?de": {
"protocol": "https:",
"hostname": "example.com",
"port": "443",
"pathname": "/a/b/c",
"search": "?de",
"full": "https://example.com:443/a/b/c?de",
},
"https://[::ffff:a9fe:a9fe]/a/b/c?de": {
"protocol": "https:",
"hostname": "::ffff:a9fe:a9fe",
"pathname": "/a/b/c",
"search": "?de",
"full": "https://[::ffff:a9fe:a9fe]/a/b/c?de",
},
"http://[::ffff:a9fe:a9fe]:80/a/b/c?de": {
"protocol": "http:",
"hostname": "::ffff:a9fe:a9fe",
"port": "80",
"pathname": "/a/b/c",
"search": "?de",
"full": "http://[::ffff:a9fe:a9fe]:80/a/b/c?de",
},
}
for url, expected in data.items():
assert get_url_dict(url) == expected
def get_data_from_request(request, capture_body=False, capture_headers=True):
result = {
"method": request.method,
"socket": {"remote_address": request.remote, "encrypted": request.secure},
"cookies": dict(request.cookies),
}
if capture_headers:
result["headers"] = dict(request.headers)
# TODO: capture body
result["url"] = get_url_dict(str(request.url))
return result
def get_data_from_request(request):
data = {
"headers": dict(**request.headers),
"method": request.method,
"socket": {
"remote_address": request.remote_addr,
"encrypted": request.scheme == 'https'
},
"cookies": dict(**request.cookies),
"url": get_url_dict(request.url)
}
# remove Cookie header since the same data is in request["cookies"] as well
data["headers"].pop("Cookie", None)
return data
def get_data_from_request(self, request, capture_body=False):
result = {
'env': dict(get_environ(request.META)),
'headers': dict(get_headers(request.META)),
'method': request.method,
'socket': {
'remote_address': request.META.get('REMOTE_ADDR'),
'encrypted': request.is_secure()
},
'cookies': dict(request.COOKIES),
}
if request.method in constants.HTTP_WITH_BODY:
content_type = request.META.get('CONTENT_TYPE')
if content_type == 'application/x-www-form-urlencoded':
data = compat.multidict_to_dict(request.POST)
elif content_type and content_type.startswith(
'multipart/form-data'):
data = compat.multidict_to_dict(request.POST)
if request.FILES:
data['_files'] = {
field: file.name
for field, file in compat.iteritems(request.FILES)
}
else:
try:
data = request.body
except Exception:
data = '<unavailable>'
result['body'] = data if (capture_body
or not data) else '[REDACTED]'
if hasattr(request, 'get_raw_uri'):
# added in Django 1.9
url = request.get_raw_uri()
else:
try:
# Requires host to be in ALLOWED_HOSTS, might throw a
# DisallowedHost exception
url = request.build_absolute_uri()
except DisallowedHost:
# We can't figure out the real URL, so we have to set it to
# DisallowedHost
result['url'] = {'full': 'DisallowedHost'}
url = None
if url:
result['url'] = get_url_dict(url)
return result
def get_data_from_request(request: Request, config: Config, event_type: str):
result = {
"method": request.method,
"socket": {
"remote_address": request.remote
},
"cookies": dict(request.cookies),
}
if config.capture_headers:
result["headers"] = dict(request.headers)
# TODO: capture body
result["url"] = get_url_dict(str(request.url))
return result
def handle_exception(self, exc_info, environ):
event_id = self.client.capture(
"Exception",
exc_info=exc_info,
context={
"request": {
"method": environ.get("REQUEST_METHOD"),
"url": get_url_dict(get_current_url(environ)),
"headers": dict(get_headers(environ)),
"env": dict(get_environ(environ)),
}
},
handled=False,
)
return event_id
def get_data_from_request(self, request):
data = {
'headers': dict(**request.headers),
'method': request.method,
'socket': {
'remote_address': request.remote_addr,
'encrypted': request.scheme == 'https',
},
'cookies': dict(**request.cookies),
'url': get_url_dict(request.url),
}
# remove Cookie header since the same data is in
# request["cookies"] as well
data['headers'].pop('Cookie', None)
return data
def handle_exception(self, exc_info, environ):
event_id = self.client.capture(
'Exception',
exc_info=exc_info,
context={
'request': {
'method': environ.get('REQUEST_METHOD'),
'url': get_url_dict(get_current_url(environ)),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
},
handled=False,
)
return event_id
def get_data_from_request(request):
url = '{}://{}{}'.format(request.protocol, request.host, request.uri)
data = {
"headers": dict(**request.headers),
"method": request.method,
"socket": {
"remote_address": request.remote_ip,
"encrypted": request.protocol == 'https'
},
"cookies": dict(**request.cookies),
"url": get_url_dict(url),
"body": request.body
}
data["headers"].pop("Cookie", None)
return data
def get_data_from_request(self, request, event_type):
result = {
"env": dict(get_environ(request.META)),
"method": request.method,
"socket": {
"remote_address": request.META.get("REMOTE_ADDR")
},
"cookies": dict(request.COOKIES),
}
if self.config.capture_headers:
request_headers = dict(get_headers(request.META))
for key, value in request_headers.items():
if isinstance(value, (int, float)):
request_headers[key] = str(value)
result["headers"] = request_headers
if request.method in constants.HTTP_WITH_BODY:
capture_body = self.config.capture_body in ("all", event_type)
if not capture_body:
result["body"] = "[REDACTED]"
else:
content_type = request.META.get("CONTENT_TYPE")
if content_type == "application/x-www-form-urlencoded":
data = compat.multidict_to_dict(request.POST)
elif content_type and content_type.startswith(
"multipart/form-data"):
data = compat.multidict_to_dict(request.POST)
if request.FILES:
data["_files"] = {
field: file.name
for field, file in compat.iteritems(request.FILES)
}
else:
try:
data = request.body
except Exception as e:
self.logger.debug("Can't capture request body: %s",
compat.text_type(e))
data = "<unavailable>"
if data is not None:
result["body"] = data
url = get_raw_uri(request)
result["url"] = get_url_dict(url)
return result
def get_data_from_request(self, request):
if request.method != 'GET':
try:
if hasattr(request, 'body'):
# Django 1.4+
raw_data = request.body
else:
raw_data = request.raw_post_data
data = raw_data if raw_data else request.POST
except Exception:
# assume we had a partial read:
data = '<unavailable>'
else:
data = None
environ = request.META
result = {
'body': data,
'env': dict(get_environ(environ)),
'headers': dict(get_headers(environ)),
'method': request.method,
'socket': {
'remote_address': request.META.get('REMOTE_ADDR'),
'encrypted': request.is_secure()
},
'cookies': dict(request.COOKIES),
}
if hasattr(request, 'get_raw_uri'):
# added in Django 1.9
url = request.get_raw_uri()
else:
try:
# Requires host to be in ALLOWED_HOSTS, might throw a
# DisallowedHost exception
url = request.build_absolute_uri()
except DisallowedHost:
# We can't figure out the real URL, so we have to set it to
# DisallowedHost
result['url'] = {'raw': 'DisallowedHost'}
url = None
if url:
result['url'] = get_url_dict(url)
return result
def set_tag(self, key, value):
if self.is_transaction:
if key == "type":
self.elastic_apm_ref.transaction_type = value
elif key == "result":
self.elastic_apm_ref.result = value
elif key == tags.HTTP_STATUS_CODE:
self.elastic_apm_ref.result = "HTTP {}xx".format(
compat.text_type(value)[0])
traces.set_context({"status_code": value}, "response")
elif key == "user.id":
traces.set_user_context(user_id=value)
elif key == "user.username":
traces.set_user_context(username=value)
elif key == "user.email":
traces.set_user_context(email=value)
elif key == tags.HTTP_URL:
traces.set_context({"url": get_url_dict(value)}, "request")
elif key == tags.HTTP_METHOD:
traces.set_context({"method": value}, "request")
elif key == tags.COMPONENT:
traces.set_context({"framework": {"name": value}}, "service")
else:
self.elastic_apm_ref.tag(**{key: value})
else:
if key.startswith("db."):
span_context = self.elastic_apm_ref.context or {}
if "db" not in span_context:
span_context["db"] = {}
if key == tags.DATABASE_STATEMENT:
span_context["db"]["statement"] = value
elif key == tags.DATABASE_USER:
span_context["db"]["user"] = value
elif key == tags.DATABASE_TYPE:
span_context["db"]["type"] = value
self.elastic_apm_ref.type = "db." + value
else:
self.elastic_apm_ref.tag(**{key: value})
self.elastic_apm_ref.context = span_context
elif key == tags.SPAN_KIND:
self.elastic_apm_ref.type = value
else:
self.elastic_apm_ref.tag(**{key: value})
return self
async def get_data_from_request(request: Request, config: Config,
event_type: str) -> dict:
"""Loads data from incoming request for APM capturing.
Args:
request (Request)
config (Config)
event_type (str)
Returns:
dict
"""
result = {
"method": request.method,
"socket": {
"remote_address": _get_client_ip(request),
"encrypted": request.url.is_secure
},
"cookies": request.cookies,
}
if config.capture_headers:
result["headers"] = dict(request.headers)
if request.method in constants.HTTP_WITH_BODY:
if config.capture_body not in ("all", event_type):
result["body"] = "[REDACTED]"
else:
body = None
try:
body = await get_body(request)
if request.headers.get(
"content-type") == "application/x-www-form-urlencoded":
body = await query_params_to_dict(body)
else:
body = json.loads(body)
except Exception:
pass
if body is not None:
result["body"] = body
result["url"] = get_url_dict(str(request.url))
return result
async def get_request_info(config: Config, request: Request) -> Dict[str, str]:
"""
Generate a traceable context information from the inbound HTTP request
:param config: Application Configuration used to tune the way the data is captured
:param request: Inbound HTTP request
:return: A dictionary containing the context information of the ongoing transaction
"""
env = dict(get_env(request=request))
app_config = {
k: v
for k, v in dict(request.app.config).items()
if all(letter in ascii_uppercase for letter in k)
}
env.update(app_config)
result = {
"env": env,
"method": request.method,
"socket": {
"remote_address": _get_client_ip(request=request),
"encrypted": request.scheme in ["https", "wss"],
},
"cookies": request.cookies,
"http_version": request.version,
}
if config.capture_headers:
result["headers"] = dict(request.headers)
if request.method in constants.HTTP_WITH_BODY and config.capture_body:
if request.content_type.startswith(
"multipart") or "octet-stream" in request.content_type:
result["body"] = "[DISCARDED]"
try:
result["body"] = request.body.decode("utf-8")
except Exception:
pass
if "body" not in result:
result["body"] = "[REDACTED]"
result["url"] = get_url_dict(request.url)
return result
async def get_request_info(
config: Config,
request: Request,
skip_headers: Optional[List[str]] = None) -> Dict[str, str]:
"""
Generate a traceable context information from the inbound HTTP request
:param config: Application Configuration used to tune the way the data is captured
:param request: Inbound HTTP request
:param skip_headers: A list of String indicating which headers to be skipped while extracting the headers
:return: A dictionary containing the context information of the ongoing transaction
"""
env = dict(get_env(request=request))
env.update(dict(request.app.config))
result = {
"env": env,
"method": request.method,
"socket": {
"remote_address": _get_client_ip(request=request),
"encrypted": request.scheme in ["https", "wss"],
},
"cookies": request.cookies,
"http_version": request.version,
}
if config.capture_headers:
result["headers"] = extract_header(entity=request,
skip_headers=skip_headers)
if request.method in constants.HTTP_WITH_BODY and config.capture_body:
if request.content_type.startswith(
"multipart") or "octet-stream" in request.content_type:
result["body"] = "[DISCARDED]"
try:
result["body"] = request.body.decode("utf-8")
except Exception:
pass
if "body" not in result:
result["body"] = "[REDACTED]"
result["url"] = get_url_dict(request.url)
return result
def test_get_url_dict():
data = {
'http://example.com': {
'protocol': 'http:',
'hostname': 'example.com',
'pathname': '',
'full': 'http://example.com',
},
'http://example.com:443': {
'protocol': 'http:',
'hostname': 'example.com',
'port': '443',
'pathname': '',
'full': 'http://example.com:443',
},
'http://example.com:443/a/b/c': {
'protocol': 'http:',
'hostname': 'example.com',
'port': '443',
'pathname': '/a/b/c',
'full': 'http://example.com:443/a/b/c',
},
'https://example.com:443/': {
'protocol': 'https:',
'hostname': 'example.com',
'port': '443',
'pathname': '/',
'full': 'https://example.com:443/',
},
'https://example.com:443/a/b/c?de': {
'protocol': 'https:',
'hostname': 'example.com',
'port': '443',
'pathname': '/a/b/c',
'search': '?de',
'full': 'https://example.com:443/a/b/c?de',
}
}
for url, expected in data.items():
assert get_url_dict(url) == expected
def get_data_from_request(request):
body = None
if request.data:
body = request.data
elif request.form:
body = urlencode(request.form)
result = {
'body': body,
'env': dict(get_environ(request.environ)),
'headers': dict(get_headers(request.environ), ),
'method': request.method,
'socket': {
'remote_address': request.environ.get('REMOTE_ADDR'),
'encrypted': request.is_secure
},
'cookies': request.cookies,
}
result['url'] = get_url_dict(request.url)
return result
async def get_data_from_request(request: Request,
capture_body=False,
capture_headers=True) -> dict:
"""Loads data from incoming request for APM capturing.
Args:
request (Request)
capture_body (bool): Loads also request body if True
capture_headers (bool): Loads also request headers if True
Returns:
dict
"""
result = {
"method": request.method,
"socket": {
"remote_address": _get_client_ip(request),
"encrypted": request.url.is_secure
},
"cookies": request.cookies,
}
if capture_headers:
result["headers"] = dict(request.headers)
if request.method in constants.HTTP_WITH_BODY:
body = await get_body(request)
if request.headers[
'content-type'] == "application/x-www-form-urlencoded":
body = await query_params_to_dict(body)
else:
body = json.loads(body)
if body is not None:
result["body"] = body if capture_body else "[REDACTED]"
result["url"] = get_url_dict(str(request.url))
return result
def test_get_url_dict(url, expected):
assert get_url_dict(url) == expected