def prepare_response(self, cached):
"""Verify our vary headers match and construct a real urllib3
HTTPResponse object.
"""
# Special case the '*' Vary value as it means we cannot actually
# determine if the cached response is suitable for this request.
if "*" in cached.get("vary", {}):
return
body_raw = cached["response"].pop("body")
headers = CaseInsensitiveDict(data=cached['response']['headers'])
if headers.get('transfer-encoding', '') == 'chunked':
headers.pop('transfer-encoding')
cached['response']['headers'] = headers
try:
body = io.BytesIO(body_raw)
except TypeError:
# This can happen if cachecontrol serialized to v1 format (pickle)
# using Python 2. A Python 2 str(byte string) will be unpickled as
# a Python 3 str (unicode string), which will cause the above to
# fail with:
#
# TypeError: 'str' does not support the buffer interface
body = io.BytesIO(body_raw.encode('utf8'))
return HTTPResponse(body=body,
preload_content=False,
**cached["response"])
def prepare_response(self, request, cached):
"""Verify our vary headers match and construct a real urllib3
HTTPResponse object.
"""
# Special case the '*' Vary value as it means we cannot actually
# determine if the cached response is suitable for this request.
if "*" in cached.get("vary", {}):
return
# Ensure that the Vary headers for the cached response match our
# request
for header, value in cached.get("vary", {}).items():
if request.headers.get(header, None) != value:
return
body_file = cached[u'response'].pop(u'body')
body = open(os.path.join(self.cache.directory, body_file), 'rb')
headers = CaseInsensitiveDict(data=cached['response']['headers'])
if headers.get('transfer-encoding', '') == 'chunked':
headers.pop('transfer-encoding')
cached['response']['headers'] = headers
return HTTPResponse(
body=body,
preload_content=False,
**cached["response"]
)
def prepare_response(self, cached):
"""Verify our vary headers match and construct a real urllib3
HTTPResponse object.
"""
# Special case the '*' Vary value as it means we cannot actually
# determine if the cached response is suitable for this request.
if "*" in cached.get("vary", {}):
return
body_raw = cached["response"].pop("body")
headers = CaseInsensitiveDict(data=cached['response']['headers'])
if headers.get('transfer-encoding', '') == 'chunked':
headers.pop('transfer-encoding')
cached['response']['headers'] = headers
try:
body = io.BytesIO(body_raw)
except TypeError:
# This can happen if cachecontrol serialized to v1 format (pickle)
# using Python 2. A Python 2 str(byte string) will be unpickled as
# a Python 3 str (unicode string), which will cause the above to
# fail with:
#
# TypeError: 'str' does not support the buffer interface
body = io.BytesIO(body_raw.encode('utf8'))
return HTTPResponse(
body=body,
preload_content=False,
**cached["response"]
)
def process_extra_headers(self, headers):
# Handle some special-case headers, and pass the remainder to set_extra_headers.
# (Subclasses shouldn't need to override this.)
headers = CaseInsensitiveDict(headers) # email headers are case-insensitive per RFC-822 et seq
reply_to = headers.pop('Reply-To', None)
if reply_to:
# message.extra_headers['Reply-To'] will override message.reply_to
# (because the extra_headers attr is processed after reply_to).
# This matches the behavior of Django's EmailMessage.message().
self.set_reply_to(parse_address_list([reply_to], field="extra_headers['Reply-To']"))
if 'From' in headers:
# If message.extra_headers['From'] is supplied, it should override message.from_email,
# but message.from_email should be used as the envelope_sender. See:
# - https://code.djangoproject.com/ticket/9214
# - https://github.com/django/django/blob/1.8/django/core/mail/message.py#L269
# - https://github.com/django/django/blob/1.8/django/core/mail/backends/smtp.py#L118
header_from = parse_address_list(headers.pop('From'), field="extra_headers['From']")
envelope_sender = parse_single_address(self.message.from_email, field="from_email") # must be single
self.set_from_email_list(header_from)
self.set_envelope_sender(envelope_sender)
if 'To' in headers:
# If message.extra_headers['To'] is supplied, message.to is used only as the envelope
# recipients (SMTP.sendmail to_addrs), and the header To is spoofed. See:
# - https://github.com/django/django/blob/1.8/django/core/mail/message.py#L270
# - https://github.com/django/django/blob/1.8/django/core/mail/backends/smtp.py#L119-L120
# No current ESP supports this, so this code is mainly here to flag
# the SMTP backend's behavior as an unsupported feature in Anymail:
header_to = headers.pop('To')
self.set_spoofed_to_header(header_to)
if headers:
self.set_extra_headers(headers)
def set_extra_headers(self, headers):
header_dict = CaseInsensitiveDict(headers)
if 'Reply-To' in header_dict:
self.data["ReplyTo"] = header_dict.pop('Reply-To')
self.data["Headers"] = [
{"Name": key, "Value": value}
for key, value in header_dict.items()
]
def set_extra_headers(self, headers):
header_dict = CaseInsensitiveDict(headers)
if 'Reply-To' in header_dict:
self.data["ReplyTo"] = header_dict.pop('Reply-To')
self.data["Headers"] = [
{"Name": key, "Value": value}
for key, value in header_dict.items()
]
def retrieve(self, request):
filename = self._fn(request.url)
resp = Response()
headers = utils.read('%s.metadata' % filename)
if headers:
try:
headers = CaseInsensitiveDict(json.loads(headers))
except ValueError:
return None
headers['x-cache'] = 'HIT from %s' % self.__class__.__name__
resp.url = headers.pop('url', None)
resp.status_code = headers.pop('status-code', None)
resp.encoding = headers.pop('encoding', None)
resp.headers = headers
resp._content = utils.read(filename)
return resp
else:
return None
def sync_recipients_to_list(self,
list_id,
recipients,
delete_recipients=True):
stats = {}
stats['create_recipients'] = self._create_recipients(recipients)
all_recipients_lookup = CaseInsensitiveDict(
{r.email: r.id
for r in self._all_recipients()})
list_recipients = list(self._list_recipients(list_id))
recipients_not_assigned_to_list_lookup = CaseInsensitiveDict({
r['email']: all_recipients_lookup[r['email']]
for r in recipients if r['email'] in all_recipients_lookup
})
for recipient in list_recipients:
recipients_not_assigned_to_list_lookup.pop(recipient.email, None)
recipient_ids_to_add = list(
recipients_not_assigned_to_list_lookup.values())
stats['add_recipients_to_list'] = self._add_recipients_to_list(
list_id, recipient_ids_to_add)
if delete_recipients:
contact_to_delete_lookup = CaseInsensitiveDict(
{c.email: c.id
for c in list_recipients})
for recipient in recipients:
contact_to_delete_lookup.pop(recipient['email'], None)
recipients_ids_to_delete = list(contact_to_delete_lookup.values())
stats[
'delete_recipient_from_list'] = self._delete_recipient_from_list(
list_id, recipients_ids_to_delete)
stats['delete_recipients'] = self._delete_recipients(
recipients_ids_to_delete)
return stats
def verify_signature(self, query_parameters):
"""Verify the signature provided with the query parameters.
http://docs.shopify.com/api/tutorials/oauth
example usage::
from shopify_trois import Credentials
from shopify_trois.engines import Json as Shopify
from urllib.parse import parse_qsl
credentials = Credentials(
api_key='your_api_key',
scope=['read_orders'],
secret='your_app_secret'
)
shopify = Shopify(shop_name="your_store_name", credentials=\
credentials)
query_parameters = parse_qsl("code=238420989938cb70a609f6ece2e2586\
b&shop=yourstore.myshopify.com×tamp=1373382939&\
signature=6fb122e33c21851c465345b8cb97245e")
if not shopify.verify_signature(query_parameters):
raise Exception("invalid signature")
credentials.code = dict(query_parameters).get('code')
shopify.setup_access_token()
:returns: Returns True if the signature is valid.
"""
params = CaseInsensitiveDict(query_parameters)
signature = params.pop("signature", None)
calculated = ["%s=%s" % (k, v) for k, v in params.items()]
calculated.sort()
calculated = "".join(calculated)
calculated = "{secret}{calculated}".format(
secret=self.credentials.secret,
calculated=calculated
)
md5 = hashlib.md5()
md5.update(calculated.encode('utf-8'))
produced = md5.hexdigest()
return produced == signature
def prepare_response(self, request, cached):
"""Verify our vary headers match and construct a real urllib3
HTTPResponse object.
"""
# Special case the '*' Vary value as it means we cannot actually
# determine if the cached response is suitable for this request.
# This case is also handled in the controller code when creating
# a cache entry, but is left here for backwards compatibility.
if "*" in cached.get("vary", {}):
return
# Ensure that the Vary headers for the cached response match our
# request
for header, value in cached.get("vary", {}).items():
if request.headers.get(header, None) != value:
return
body_raw = cached["response"].pop("body")
headers = CaseInsensitiveDict(data=cached["response"]["headers"])
if headers.get("transfer-encoding", "") == "chunked":
headers.pop("transfer-encoding")
cached["response"]["headers"] = headers
try:
body = io.BytesIO(body_raw)
except TypeError:
# This can happen if cachecontrol serialized to v1 format (pickle)
# using Python 2. A Python 2 str(byte string) will be unpickled as
# a Python 3 str (unicode string), which will cause the above to
# fail with:
#
# TypeError: 'str' does not support the buffer interface
body = io.BytesIO(body_raw.encode("utf8"))
return HTTPResponse(body=body,
preload_content=False,
**cached["response"])
def verify_signature(self, query_parameters):
"""Verify the signature provided with the query parameters.
http://docs.shopify.com/api/tutorials/oauth
example usage::
from shopify_trois import Credentials
from shopify_trois.engines import Json as Shopify
from urllib.parse import parse_qsl
credentials = Credentials(
api_key='your_api_key',
scope=['read_orders'],
secret='your_app_secret'
)
shopify = Shopify(shop_name="your_store_name", credentials=\
credentials)
query_parameters = parse_qsl("code=238420989938cb70a609f6ece2e2586\
b&shop=yourstore.myshopify.com×tamp=1373382939&\
signature=6fb122e33c21851c465345b8cb97245e")
if not shopify.verify_signature(query_parameters):
raise Exception("invalid signature")
credentials.code = dict(query_parameters).get('code')
shopify.setup_access_token()
:returns: Returns True if the signature is valid.
"""
params = CaseInsensitiveDict(query_parameters)
signature = params.pop("signature", None)
calculated = ["%s=%s" % (k, v) for k, v in params.items()]
calculated.sort()
calculated = "".join(calculated)
calculated = "{secret}{calculated}".format(
secret=self.credentials.secret, calculated=calculated)
md5 = hashlib.md5()
md5.update(calculated.encode('utf-8'))
produced = md5.hexdigest()
return produced == signature
def prepare_response(self, request, cached):
"""Verify our vary headers match and construct a real urllib3
HTTPResponse object.
"""
# Special case the '*' Vary value as it means we cannot actually
# determine if the cached response is suitable for this request.
# This case is also handled in the controller code when creating
# a cache entry, but is left here for backwards compatibility.
if "*" in cached.get("vary", {}):
return
# Ensure that the Vary headers for the cached response match our
# request
for header, value in cached.get("vary", {}).items():
if request.headers.get(header, None) != value:
return
body_raw = cached["response"].pop("body")
headers = CaseInsensitiveDict(data=cached["response"]["headers"])
if headers.get("transfer-encoding", "") == "chunked":
headers.pop("transfer-encoding")
cached["response"]["headers"] = headers
try:
body = io.BytesIO(body_raw)
except TypeError:
# This can happen if cachecontrol serialized to v1 format (pickle)
# using Python 2. A Python 2 str(byte string) will be unpickled as
# a Python 3 str (unicode string), which will cause the above to
# fail with:
#
# TypeError: 'str' does not support the buffer interface
body = io.BytesIO(body_raw.encode("utf8"))
return HTTPResponse(body=body, preload_content=False, **cached["response"])
def __request_callback(self, request, uri, response_headers):
self.__readNextRequest(self.__cnx.verb, self.__cnx.url,
self.__cnx.input, self.__cnx.headers)
status = int(readLine(self.__file))
self.response_headers = CaseInsensitiveDict(eval(readLine(
self.__file)))
output = bytearray(readLine(self.__file), "utf-8")
readLine(self.__file)
# make a copy of the headers and remove the ones that interfere with the response handling
adding_headers = CaseInsensitiveDict(self.response_headers)
adding_headers.pop("content-length", None)
adding_headers.pop("transfer-encoding", None)
adding_headers.pop("content-encoding", None)
response_headers.update(adding_headers)
return [status, response_headers, output]
def __request_callback(self, request, uri, response_headers):
self.__readNextRequest(self.__cnx.verb, self.__cnx.url, self.__cnx.input, self.__cnx.headers)
status = int(readLine(self.__file))
self.response_headers = CaseInsensitiveDict(eval(readLine(self.__file)))
output = readLine(self.__file)
readLine(self.__file)
if atLeastPython3:
output = bytes(output, 'utf-8')
# make a copy of the headers and remove the ones that interfere with the response handling
adding_headers = CaseInsensitiveDict(self.response_headers)
adding_headers.pop('content-length', None)
adding_headers.pop('transfer-encoding', None)
adding_headers.pop('content-encoding', None)
response_headers.update(adding_headers)
return [status, response_headers, output]
def reload_config(self, config):
self._superuser = config['authentication'].get('superuser', {})
server_parameters = self.get_server_parameters(config)
conf_changed = hba_changed = ident_changed = local_connection_address_changed = pending_restart = False
if self._postgresql.state == 'running':
changes = CaseInsensitiveDict({p: v for p, v in server_parameters.items() if '.' not in p})
changes.update({p: None for p in self._server_parameters.keys() if not ('.' in p or p in changes)})
if changes:
# XXX: query can raise an exception
for r in self._postgresql.query(('SELECT name, setting, unit, vartype, context '
+ 'FROM pg_catalog.pg_settings ' +
' WHERE pg_catalog.lower(name) IN ('
+ ', '.join(['%s'] * len(changes)) +
')'), *(k.lower() for k in changes.keys())):
if r[4] != 'internal' and r[0] in changes:
new_value = changes.pop(r[0])
if new_value is None or not compare_values(r[3], r[2], r[1], new_value):
if r[4] == 'postmaster':
pending_restart = True
logger.info('Changed %s from %s to %s (restart required)', r[0], r[1], new_value)
if config.get('use_unix_socket') and r[0] == 'unix_socket_directories'\
or r[0] in ('listen_addresses', 'port'):
local_connection_address_changed = True
else:
logger.info('Changed %s from %s to %s', r[0], r[1], new_value)
conf_changed = True
for param in changes:
if param in server_parameters:
logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
server_parameters.pop(param)
# Check that user-defined-paramters have changed (parameters with period in name)
if not conf_changed:
for p, v in server_parameters.items():
if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])):
logger.info('Changed %s from %s to %s', p, self._server_parameters.get(p), v)
conf_changed = True
break
if not conf_changed:
for p, v in self._server_parameters.items():
if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])):
logger.info('Changed %s from %s to %s', p, v, server_parameters.get(p))
conf_changed = True
break
if not server_parameters.get('hba_file') and config.get('pg_hba'):
hba_changed = self._config.get('pg_hba', []) != config['pg_hba']
if not server_parameters.get('ident_file') and config.get('pg_ident'):
ident_changed = self._config.get('pg_ident', []) != config['pg_ident']
self._config = config
self._postgresql.set_pending_restart(pending_restart)
self._server_parameters = server_parameters
self._adjust_recovery_parameters()
self._connect_address = config.get('connect_address')
self._krbsrvname = config.get('krbsrvname')
# for not so obvious connection attempts that may happen outside of pyscopg2
if self._krbsrvname:
os.environ['PGKRBSRVNAME'] = self._krbsrvname
if not local_connection_address_changed:
self.resolve_connection_addresses()
if conf_changed:
self.write_postgresql_conf()
if hba_changed:
self.replace_pg_hba()
if ident_changed:
self.replace_pg_ident()
if conf_changed or hba_changed or ident_changed:
logger.info('PostgreSQL configuration items changed, reloading configuration.')
self._postgresql.reload()
elif not pending_restart:
logger.info('No PostgreSQL configuration items changed, nothing to reload.')
def reload_config(self, config, sighup=False):
self._superuser = config['authentication'].get('superuser', {})
server_parameters = self.get_server_parameters(config)
conf_changed = hba_changed = ident_changed = local_connection_address_changed = pending_restart = False
if self._postgresql.state == 'running':
changes = CaseInsensitiveDict(
{p: v
for p, v in server_parameters.items() if '.' not in p})
changes.update({
p: None
for p in self._server_parameters.keys()
if not ('.' in p or p in changes)
})
if changes:
if 'wal_buffers' in changes: # we need to calculate the default value of wal_buffers
undef = [
p for p in ('shared_buffers', 'wal_segment_size',
'wal_block_size') if p not in changes
]
changes.update({p: None for p in undef})
# XXX: query can raise an exception
old_values = {
r[0]: r
for r in self._postgresql.query((
'SELECT name, setting, unit, vartype, context ' +
'FROM pg_catalog.pg_settings ' +
' WHERE pg_catalog.lower(name) = ANY(%s)'
), [k.lower() for k in changes.keys()])
}
if 'wal_buffers' in changes:
self._handle_wal_buffers(old_values, changes)
for p in undef:
del changes[p]
for r in old_values.values():
if r[4] != 'internal' and r[0] in changes:
new_value = changes.pop(r[0])
if new_value is None or not compare_values(
r[3], r[2], r[1], new_value):
conf_changed = True
if r[4] == 'postmaster':
pending_restart = True
logger.info(
'Changed %s from %s to %s (restart might be required)',
r[0], r[1], new_value)
if config.get('use_unix_socket') and r[0] == 'unix_socket_directories'\
or r[0] in ('listen_addresses', 'port'):
local_connection_address_changed = True
else:
logger.info('Changed %s from %s to %s', r[0],
r[1], new_value)
for param in changes:
if param in server_parameters:
logger.warning(
'Removing invalid parameter `%s` from postgresql.parameters',
param)
server_parameters.pop(param)
# Check that user-defined-paramters have changed (parameters with period in name)
for p, v in server_parameters.items():
if '.' in p and (p not in self._server_parameters
or str(v) != str(self._server_parameters[p])):
logger.info('Changed %s from %s to %s', p,
self._server_parameters.get(p), v)
conf_changed = True
for p, v in self._server_parameters.items():
if '.' in p and (p not in server_parameters
or str(v) != str(server_parameters[p])):
logger.info('Changed %s from %s to %s', p, v,
server_parameters.get(p))
conf_changed = True
if not server_parameters.get('hba_file') and config.get('pg_hba'):
hba_changed = self._config.get('pg_hba',
[]) != config['pg_hba']
if not server_parameters.get('ident_file') and config.get(
'pg_ident'):
ident_changed = self._config.get('pg_ident',
[]) != config['pg_ident']
self._config = config
self._postgresql.set_pending_restart(pending_restart)
self._server_parameters = server_parameters
self._adjust_recovery_parameters()
self._krbsrvname = config.get('krbsrvname')
# for not so obvious connection attempts that may happen outside of pyscopg2
if self._krbsrvname:
os.environ['PGKRBSRVNAME'] = self._krbsrvname
if not local_connection_address_changed:
self.resolve_connection_addresses()
if conf_changed:
self.write_postgresql_conf()
if hba_changed:
self.replace_pg_hba()
if ident_changed:
self.replace_pg_ident()
if sighup or conf_changed or hba_changed or ident_changed:
logger.info('Reloading PostgreSQL configuration.')
self._postgresql.reload()
if self._postgresql.major_version >= 90500:
time.sleep(1)
try:
pending_restart = self._postgresql.query(
'SELECT COUNT(*) FROM pg_catalog.pg_settings'
' WHERE pending_restart').fetchone()[0] > 0
self._postgresql.set_pending_restart(pending_restart)
except Exception as e:
logger.warning('Exception %r when running query', e)
else:
logger.info(
'No PostgreSQL configuration items changed, nothing to reload.'
)
def sniffit():
"""
Perform an HTTP/HTTPS request to the address that the user specifid
:return:
TODO Make the Google Verification a separate module with annotion
"""
parsed_url = urlparse(request.json["url"])
app.logger.info(request.remote_addr + " " + parsed_url.netloc)
# Processing the headers to be sent to the URL that the user defined in the interface.
# What we are doing here is making sure the the user can't override some headers that we want to force such as
# X-Forwarded-For.
request_headers = CaseInsensitiveDict({header["key"]: header["value"] for header in request.json["headers"]})
request_headers["X-Forwarded-For"] = request.remote_addr
request_headers["X-Anti-Abuse"] = app.config.get("ABUSE_CONTACT")
request_headers = {string.capwords(k, "-"): v for (k, v) in request_headers.items()}
# Request Parameters
if type(request.json["parameters"]) is list:
request_parameters = "&".join([cgi.escape(header["key"])+"="+cgi.escape(header["value"]) for header in request.json["parameters"]])
else:
request_parameters = request.json["parameters"]
# Base Response JSON
response_json = {'success': False, 'sniffed': None, 'messages': []}
try:
if string.lower(request.json["method"]) in ["get", "head", "options"]:
response = requests.request(request.json["method"], request.json["url"], verify=False,
params=request_parameters, headers=request_headers)
else:
response = requests.request(request.json["method"], request.json["url"],
verify=False, data=request_parameters, headers=request_headers)
# I prefer to have the capitalized headers in the frontend
# This will convert the headers from 'content-type' to 'Content-Type'
response_headers = {string.capwords(k, "-"): v for (k, v) in response.headers.items()}
# This is for the adrministrators only so there is no need for the end-user to see this
request_headers.pop("X-Anti-Abuse")
request_headers.pop("X-Forwarded-For")
# Create a history of redirects to inform the user
redirections = [{"url": redirect.headers["location"]} for redirect in response.history]
# Geo Location
ipaddress = socket.gethostbyname(parsed_url.netloc)
geolocation_response = requests.get("http://ip-api.com/json/" + ipaddress);
response_json["success"] = True
response_json["showRecaptcha"] = recaptcha_handler.is_token_invalid()
response_json["sniffed"] = {
'headers': {
'response': response_headers,
'request': request_headers
},
'ipaddress': ipaddress,
'geolocation': geolocation_response.json(),
'ssl': None,
'redirect': redirections,
'body': base64.b64encode(cgi.escape(response.text.encode("UTF-8"))),
'size': response.headers.get("content-length", False),
'ssize': len(response.text.encode("UTF-8")),
'elapsed': response.elapsed.total_seconds(),
'status': {
"reason": response.reason,
"code": str(response.status_code)
}
}
except Exception as e:
raise RequestFailedException(repr(e))
return jsonify(response_json)
class HttpRequest(object):
"""Represents a HTTP request.
URL can be given without query parameters, to be added later using "format_parameters".
:param str method: HTTP method (GET, HEAD, etc.)
:param str url: At least complete scheme/host/path
:param dict[str,str] headers: HTTP headers
:param files: Files list.
:param data: Body to be sent.
:type data: bytes or str.
"""
def __init__(self, method, url, headers=None, files=None, data=None):
# type: (str, str, Mapping[str, str], Any, Any) -> None
self.method = method
self.url = url
self.headers = CaseInsensitiveDict(headers)
self.files = files
self.data = data
def __repr__(self):
return '<HttpRequest [%s]>' % (self.method)
@property
def query(self):
"""The query parameters of the request as a dict."""
query = urlparse(self.url).query
if query:
return {
p[0]: p[-1]
for p in [p.partition('=') for p in query.split('&')]
}
return {}
@property
def body(self):
"""Alias to data."""
return self.data
@body.setter
def body(self, value):
self.data = value
@staticmethod
def _format_data(data):
# type: (Union[str, IO]) -> Union[Tuple[None, str], Tuple[Optional[str], IO, str]]
"""Format field data according to whether it is a stream or
a string for a form-data request.
:param data: The request field data.
:type data: str or file-like object.
"""
if hasattr(data, 'read'):
data = cast(IO, data)
data_name = None
try:
if data.name[0] != '<' and data.name[-1] != '>':
data_name = os.path.basename(data.name)
except (AttributeError, TypeError):
pass
return (data_name, data, "application/octet-stream")
return (None, cast(str, data))
def format_parameters(self, params):
# type: (Dict[str, str]) -> None
"""Format parameters into a valid query string.
It's assumed all parameters have already been quoted as
valid URL strings.
:param dict params: A dictionary of parameters.
"""
query = urlparse(self.url).query
if query:
self.url = self.url.partition('?')[0]
existing_params = {
p[0]: p[-1]
for p in [p.partition('=') for p in query.split('&')]
}
params.update(existing_params)
query_params = ["{}={}".format(k, v) for k, v in params.items()]
query = '?' + '&'.join(query_params)
self.url = self.url + query
def set_streamed_data_body(self, data):
"""Set a streamable data body.
:param data: The request field data.
"""
if not isinstance(data, binary_type) and \
not any(hasattr(data, attr) for attr in ["read", "__iter__", "__aiter__"]):
raise TypeError(
"A streamable data source must be an open file-like object or iterable."
)
self.data = data
self.files = None
def set_xml_body(self, data):
"""Set an XML element tree as the body of the request.
:param data: The request field data.
"""
if data is None:
self.data = None
else:
bytes_data = ET.tostring(data, encoding="utf8")
self.data = bytes_data.replace(b"encoding='utf8'",
b"encoding='utf-8'")
self.headers['Content-Length'] = str(len(self.data))
self.files = None
def set_json_body(self, data):
"""Set a JSON-friendly object as the body of the request.
:param data: The request field data.
"""
if data is None:
self.data = None
else:
self.data = json.dumps(data)
self.headers['Content-Length'] = str(len(self.data))
self.files = None
def set_formdata_body(self, data=None):
"""Set form-encoded data as the body of the request.
:param data: The request field data.
"""
if data is None:
data = {}
content_type = self.headers.pop('Content-Type',
None) if self.headers else None
if content_type and content_type.lower(
) == 'application/x-www-form-urlencoded':
self.data = {f: d for f, d in data.items() if d is not None}
self.files = None
else: # Assume "multipart/form-data"
self.files = {
f: self._format_data(d)
for f, d in data.items() if d is not None
}
self.data = None
def set_bytes_body(self, data):
"""Set generic bytes as the body of the request.
:param data: The request field data.
"""
if data:
self.headers['Content-Length'] = str(len(data))
self.data = data
self.files = None
class ClientRequest(object):
"""Represents a HTTP request.
URL can be given without query parameters, to be added later using "format_parameters".
Instance can be created without data, to be added later using "add_content"
Instance can be created without files, to be added later using "add_formdata"
:param str method: HTTP method (GET, HEAD, etc.)
:param str url: At least complete scheme/host/path
:param dict[str,str] headers: HTTP headers
:param files: Files list.
:param data: Body to be sent.
:type data: bytes or str.
"""
def __init__(self, method, url, headers=None, files=None, data=None):
# type: (str, str, Mapping[str, str], Any, Any) -> None
self.method = method
self.url = url
self.headers = CaseInsensitiveDict(headers)
self.files = files
self.data = data
def __repr__(self):
return '<ClientRequest [%s]>' % (self.method)
@property
def body(self):
"""Alias to data."""
return self.data
@body.setter
def body(self, value):
self.data = value
def format_parameters(self, params):
# type: (Dict[str, str]) -> None
"""Format parameters into a valid query string.
It's assumed all parameters have already been quoted as
valid URL strings.
:param dict params: A dictionary of parameters.
"""
query = urlparse(self.url).query
if query:
self.url = self.url.partition('?')[0]
existing_params = {
p[0]: p[-1]
for p in [p.partition('=') for p in query.split('&')]
}
params.update(existing_params)
query_params = ["{}={}".format(k, v) for k, v in params.items()]
query = '?' + '&'.join(query_params)
self.url = self.url + query
def add_content(self, data):
# type: (Optional[Union[Dict[str, Any], ET.Element]]) -> None
"""Add a body to the request.
:param data: Request body data, can be a json serializable
object (e.g. dictionary) or a generator (e.g. file data).
"""
if data is None:
return
if isinstance(data, ET.Element):
bytes_data = ET.tostring(data, encoding="utf8")
self.headers['Content-Length'] = str(len(bytes_data))
self.data = bytes_data
return
# By default, assume JSON
try:
self.data = json.dumps(data)
self.headers['Content-Length'] = str(len(self.data))
except TypeError:
self.data = data
@staticmethod
def _format_data(data):
# type: (Union[str, IO]) -> Union[Tuple[None, str], Tuple[Optional[str], IO, str]]
"""Format field data according to whether it is a stream or
a string for a form-data request.
:param data: The request field data.
:type data: str or file-like object.
"""
if hasattr(data, 'read'):
data = cast(IO, data)
data_name = None
try:
if data.name[0] != '<' and data.name[-1] != '>':
data_name = os.path.basename(data.name)
except (AttributeError, TypeError):
pass
return (data_name, data, "application/octet-stream")
return (None, cast(str, data))
def add_formdata(self, content=None):
# type: (Optional[Dict[str, str]]) -> None
"""Add data as a multipart form-data request to the request.
We only deal with file-like objects or strings at this point.
The requests is not yet streamed.
:param dict headers: Any headers to add to the request.
:param dict content: Dictionary of the fields of the formdata.
"""
if content is None:
content = {}
content_type = self.headers.pop('Content-Type',
None) if self.headers else None
if content_type and content_type.lower(
) == 'application/x-www-form-urlencoded':
# Do NOT use "add_content" that assumes input is JSON
self.data = {f: d for f, d in content.items() if d is not None}
else: # Assume "multipart/form-data"
self.files = {
f: self._format_data(d)
for f, d in content.items() if d is not None
}
def proxy(self):
pass
# special case: process info request
if self.path == '/headers':
response = json.dumps(
{key.lower(): value
for key, value in self.headers.items()}).encode('UTF8')
self.send_response(200)
self.send_header('content-type', 'application/json')
self.send_header('content-length', str(len(response)))
self.end_headers()
self.wfile.write(response)
return
# process request urls
c_mode, c_target = self.process_url(self.path)
p_mode, p_target = self.process_url(self.headers['referer'])
parsed = urlsplit(c_target)
# handle invalid requests
if c_mode is None or c_target is None:
if p_mode is None or p_target is None:
self.send_error(400)
else:
self.send_response(307)
self.send_header(
'location',
'/' + p_mode + '/' + urljoin(p_target, self.path))
self.send_header('vary', 'referer')
self.send_header('content-length', '0')
self.end_headers()
return
# process request body
content = None
if 'content-length' in self.headers:
content = self.rfile.read(int(self.headers['content-length']))
# process request headers
headers = CaseInsensitiveDict(self.headers)
headers.pop('user-agent', None)
headers.pop('accept-encoding', None)
headers.pop('te', None)
headers.pop('connection', None)
headers.pop('host', None)
headers['origin'] = parsed.scheme + '://' + parsed.netloc
headers['referer'] = p_target
# process request cookies
cookies = {
key: value.value
for key, value in SimpleCookie(headers.pop('cookie',
None)).items()
}
cookies.pop('__cfduid', None)
cookies.pop('cf_clearance', None)
# send upstream request
resp = ChiselSession().request(
method=self.command,
url=c_target,
data=content,
headers=headers,
cookies=cookies,
allow_redirects=False,
)
if resp is None:
self.send_error(502)
return
# send initial response
self.send_response(resp.status_code)
for keep in ('set-cookie', 'vary'):
if keep in resp.headers:
self.send_header(keep, resp.headers[keep])
if 'location' in resp.headers:
self.send_header(
'location', '/' + c_mode + '/' +
urljoin(c_target, resp.headers['location']))
# end for HEAD requests
if self.command == 'HEAD':
self.end_headers()
return
body = resp.content
# process response body
if resp.headers['content-type'].startswith('text/html'):
if c_mode == 'browser':
soup = self.make_tasty_soup(resp, True)
for tag in soup('script'):
if tag.string:
tag.string = self.expand_urls_in_text(
tag.string, parsed.scheme)
with open('intercept.js', 'r') as fp:
tag = soup.new_tag('script')
tag.append(fp.read())
soup.insert(0, tag)
body = soup.encode()
else:
body = self.make_tasty_soup(resp, False).encode()
resp.headers['content-length'] = str(len(body))
elif c_mode == 'browser' and resp.headers['content-type'].startswith(
'text/') and (resp.encoding or resp.apparent_encoding):
body = self.expand_urls_in_text(resp.text, parsed.scheme).encode(
resp.encoding or resp.apparent_encoding)
resp.headers['content-length'] = str(len(body))
# send response body and related headers
self.send_header('content-type', resp.headers['content-type'])
self.send_header('content-length', resp.headers['content-length'])
self.end_headers()
self.wfile.write(body)
class StateMachine(object):
""" Helper class that tracks the state of different entities. """
def __init__(self, bus):
self._states = CaseInsensitiveDict()
self._bus = bus
self._lock = threading.Lock()
def entity_ids(self, domain_filter=None):
""" List of entity ids that are being tracked. """
if domain_filter is not None:
domain_filter = domain_filter.lower()
return [
state.entity_id for key, state in self._states.lower_items()
if util.split_entity_id(key)[0] == domain_filter
]
else:
return list(self._states.keys())
def all(self):
""" Returns a list of all states. """
return [state.copy() for state in self._states.values()]
def get(self, entity_id):
""" Returns the state of the specified entity. """
state = self._states.get(entity_id)
# Make a copy so people won't mutate the state
return state.copy() if state else None
def get_since(self, point_in_time):
"""
Returns all states that have been changed since point_in_time.
"""
point_in_time = util.strip_microseconds(point_in_time)
with self._lock:
return [
state for state in self._states.values()
if state.last_updated >= point_in_time
]
def is_state(self, entity_id, state):
""" Returns True if entity exists and is specified state. """
return (entity_id in self._states
and self._states[entity_id].state == state)
def remove(self, entity_id):
""" Removes an entity from the state machine.
Returns boolean to indicate if an entity was removed. """
with self._lock:
return self._states.pop(entity_id, None) is not None
def set(self, entity_id, new_state, attributes=None):
""" Set the state of an entity, add entity if it does not exist.
Attributes is an optional dict to specify attributes of this state.
If you just update the attributes and not the state, last changed will
not be affected.
"""
new_state = str(new_state)
attributes = attributes or {}
with self._lock:
old_state = self._states.get(entity_id)
is_existing = old_state is not None
same_state = is_existing and old_state.state == new_state
same_attr = is_existing and old_state.attributes == attributes
# If state did not exist or is different, set it
if not (same_state and same_attr):
last_changed = old_state.last_changed if same_state else None
state = self._states[entity_id] = \
State(entity_id, new_state, attributes, last_changed)
event_data = {'entity_id': entity_id, 'new_state': state}
if old_state:
event_data['old_state'] = old_state
self._bus.fire(EVENT_STATE_CHANGED, event_data)
def track_change(self, entity_ids, action, from_state=None, to_state=None):
"""
Track specific state changes.
entity_ids, from_state and to_state can be string or list.
Use list to match multiple.
Returns the listener that listens on the bus for EVENT_STATE_CHANGED.
Pass the return value into hass.bus.remove_listener to remove it.
"""
from_state = _process_match_param(from_state)
to_state = _process_match_param(to_state)
# Ensure it is a lowercase list with entity ids we want to match on
if isinstance(entity_ids, str):
entity_ids = (entity_ids.lower(), )
else:
entity_ids = tuple(entity_id.lower() for entity_id in entity_ids)
@ft.wraps(action)
def state_listener(event):
""" The listener that listens for specific state changes. """
if event.data['entity_id'].lower() in entity_ids and \
'old_state' in event.data and \
_matcher(event.data['old_state'].state, from_state) and \
_matcher(event.data['new_state'].state, to_state):
action(event.data['entity_id'], event.data['old_state'],
event.data['new_state'])
self._bus.listen(EVENT_STATE_CHANGED, state_listener)
return state_listener
def sniffit():
"""
Perform an HTTP/HTTPS request to the address that the user specifid
:return:
TODO Make the Google Verification a separate module with annotion
"""
parsed_url = urlparse(request.json["url"])
app.logger.info(request.remote_addr + " " + parsed_url.netloc)
# Processing the headers to be sent to the URL that the user defined in the interface.
# What we are doing here is making sure the the user can't override some headers that we want to force such as
# X-Forwarded-For.
request_headers = CaseInsensitiveDict(
{header["key"]: header["value"]
for header in request.json["headers"]})
request_headers["X-Forwarded-For"] = request.remote_addr
request_headers["X-Anti-Abuse"] = app.config.get("ABUSE_CONTACT")
request_headers = {
string.capwords(k, "-"): v
for (k, v) in request_headers.items()
}
# Request Parameters
if type(request.json["parameters"]) is list:
request_parameters = "&".join([
cgi.escape(header["key"]) + "=" + cgi.escape(header["value"])
for header in request.json["parameters"]
])
else:
request_parameters = request.json["parameters"]
# Base Response JSON
response_json = {'success': False, 'sniffed': None, 'messages': []}
try:
if string.lower(request.json["method"]) in ["get", "head", "options"]:
response = requests.request(request.json["method"],
request.json["url"],
verify=False,
params=request_parameters,
headers=request_headers)
else:
response = requests.request(request.json["method"],
request.json["url"],
verify=False,
data=request_parameters,
headers=request_headers)
# I prefer to have the capitalized headers in the frontend
# This will convert the headers from 'content-type' to 'Content-Type'
response_headers = {
string.capwords(k, "-"): v
for (k, v) in response.headers.items()
}
# This is for the adrministrators only so there is no need for the end-user to see this
request_headers.pop("X-Anti-Abuse")
request_headers.pop("X-Forwarded-For")
# Create a history of redirects to inform the user
redirections = [{
"url": redirect.headers["location"]
} for redirect in response.history]
# Geo Location
ipaddress = socket.gethostbyname(parsed_url.netloc)
geolocation_response = requests.get("http://ip-api.com/json/" +
ipaddress)
response_json["success"] = True
response_json["showRecaptcha"] = recaptcha_handler.is_token_invalid()
response_json["sniffed"] = {
'headers': {
'response': response_headers,
'request': request_headers
},
'ipaddress': ipaddress,
'geolocation': geolocation_response.json(),
'ssl': None,
'redirect': redirections,
'body':
base64.b64encode(cgi.escape(response.text.encode("UTF-8"))),
'size': response.headers.get("content-length", False),
'ssize': len(response.text.encode("UTF-8")),
'elapsed': response.elapsed.total_seconds(),
'status': {
"reason": response.reason,
"code": str(response.status_code)
}
}
except Exception as e:
raise RequestFailedException(repr(e))
return jsonify(response_json)
def get_tasks(sp_result: dict) -> dict:
out_params = CaseInsensitiveDict(sp_result['OutputParameters'])
post_sp = out_params.pop(post_sp_outparam, None)
post_sp_args = json.loads(out_params.pop(post_sp_args_outparam, '{}'))
if post_sp:
post_url = urljoin(task_sp_url, '../' + post_sp)
post_sp_args.update(out_params)
else:
post_url = None
post_sp_args = None
if db_type and isinstance(db_type,
str) and db_type[:3].lower() == 'ora':
result_model = 'DictOfList'
else:
result_model = 'SqlTvp'
serial_tasks = []
for rs in sp_result['ResultSets']:
parallel_tasks = []
for row in rs:
task = CaseInsensitiveDict(row)
mdx_query = task.get(mdx_column)
if not mdx_query:
if parallel_tasks:
continue # skip a row if mdx_column is missing from a subsequent row
else:
break # skip the whole resultset if mdx_column is missing from the first row
callback_sp = task.get(callback_sp_column)
if callback_sp:
column_map = json.loads(task.get(column_map_column))
callback_url = urljoin(task_sp_url, '../' + callback_sp)
callback_args = json.loads(
task.get(callback_args_column, '{}'))
if out_params:
callback_args.update(out_params)
parallel_tasks.append({
"(://)": _url_mdx_reader,
"(...)": {
"connection_string": mdx_conn_str,
"command_text": mdx_query,
"result_model": result_model,
"column_mapping": column_map,
"pass_result_to_url": callback_url,
"more_args": callback_args
},
"(:!!)": timeout
})
if parallel_tasks:
serial_tasks.append({"[###]": parallel_tasks})
if serial_tasks:
if len(serial_tasks) == 1:
svc_grp = serial_tasks[0]
else:
svc_grp = {"[+++]": serial_tasks}
else:
svc_grp = None
return (svc_grp, post_url, post_sp_args)
class APICall(DataContainer):
HEADER_AUTHORIZATION = "Authorization"
HEADER_REAL_IP = "X-Real-IP"
HEADER_FORWARDED_FOR = "X-Forwarded-For"
""" Standard headers """
_transaction_headers = _get_headers("Trx")
""" Transaction ID """
_redacted_headers = {
HEADER_AUTHORIZATION: " ",
"Cookie": "=",
}
""" Headers whose value should be redacted. Maps header name to partition char """
@property
def HEADER_TRANSACTION(self):
return self._transaction_headers[0]
_client_headers = _get_headers("Client")
""" Client """
@property
def HEADER_CLIENT(self):
return self._client_headers[0]
_worker_headers = _get_headers("Worker")
""" Worker (machine) ID """
@property
def HEADER_WORKER(self):
return self._worker_headers[0]
_impersonate_as_headers = _get_headers("Impersonate-As")
""" Impersonate as someone else (using his identity and permissions) """
@property
def HEADER_IMPERSONATE_AS(self):
return self._impersonate_as_headers[0]
_act_as_headers = _get_headers("Act-As")
""" Act as someone else (using his identity, but with your own role and permissions) """
@property
def HEADER_ACT_AS(self):
return self._act_as_headers[0]
_async_headers = _get_headers("Async")
""" Specifies that this call should be done asynchronously """
@property
def HEADER_ASYNC(self):
return self._async_headers[0]
def __init__(
self,
endpoint_name,
remote_addr=None,
endpoint_version: PartialVersion = PartialVersion("1.0"),
data=None,
batched_data=None,
headers=None,
files=None,
trx=None,
host=None,
auth_cookie=None,
):
super().__init__(data=data, batched_data=batched_data)
self._id = database.utils.id()
self._files = files # currently dic of key to flask's FileStorage)
self._start_ts = time.time()
self._end_ts = 0
self._duration = 0
self._endpoint_name = endpoint_name
self._remote_addr = remote_addr
assert isinstance(endpoint_version, PartialVersion), endpoint_version
self._requested_endpoint_version = endpoint_version
self._actual_endpoint_version = None
self._headers = CaseInsensitiveDict()
self._kpis = {}
self._log_api = True
if headers:
self._headers.update(headers)
self._result = APICallResult()
self._auth = None
self._impersonation = None
if trx:
self.set_header(self._transaction_headers, trx)
self._requires_authorization = True
self._host = host
self._auth_cookie = auth_cookie
self._json_flags = {}
@property
def files(self):
return self._files
@property
def id(self):
return self._id
@property
def requires_authorization(self):
return self._requires_authorization
@requires_authorization.setter
def requires_authorization(self, value):
self._requires_authorization = value
@property
def log_api(self):
return self._log_api
@log_api.setter
def log_api(self, value):
self._log_api = value
def assign_new_id(self):
self._id = database.utils.id()
def get_header(self, header, default=None):
"""
Get header value
:param header: Header name options (more than on supported, listed by priority)
:param default: Default value if no such headers were found
"""
for option in header if isinstance(header,
(tuple, list)) else (header, ):
if option in self._headers:
return self._headers[option]
return default
def clear_header(self, header):
"""
Clear header value
:param header: Header name options (more than on supported, all will be cleared)
"""
for value in header if isinstance(header,
(tuple, list)) else (header, ):
self._headers.pop(value, None)
def set_header(self, header, value):
"""
Set header value
:param header: header name (if a list is provided, first item is used)
:param value: Value to set
:return:
"""
self._headers[header[0] if isinstance(header,
(tuple,
list)) else header] = value
@property
def real_ip(self):
""" Obtain visitor's IP address """
return (self.get_header(self.HEADER_FORWARDED_FOR)
or self.get_header(self.HEADER_REAL_IP) or self._remote_addr
or "untrackable")
@property
def failed(self):
return self.result and self.result.code != 200
@property
def duration(self):
return self._duration
@property
def endpoint_name(self):
return self._endpoint_name
@property
def requested_endpoint_version(self) -> PartialVersion:
return self._requested_endpoint_version
@property
def auth(self):
""" Authenticated payload (Token or Basic) """
return self._auth
@auth.setter
def auth(self, value):
self._auth = value
@property
def impersonation_headers(self):
return {
k: v
for k, v in self._headers.items()
if k in (self._impersonate_as_headers + self._act_as_headers)
}
@property
def impersonate_as(self):
return self.get_header(self._impersonate_as_headers)
@property
def act_as(self):
return self.get_header(self._act_as_headers)
@property
def impersonation(self):
return self._impersonation
@impersonation.setter
def impersonation(self, value):
self._impersonation = value
@property
def identity(self):
if self.impersonation:
if not self.impersonation.identity:
raise Exception("Missing impersonate identity")
return self.impersonation.identity
if self.auth:
if not self.auth.identity:
raise Exception(
"Missing authorized identity (not authorized?)")
return self.auth.identity
raise MissingIdentity("Missing identity")
@property
def actual_endpoint_version(self):
return self._actual_endpoint_version
@actual_endpoint_version.setter
def actual_endpoint_version(self, value):
self._actual_endpoint_version = value
@property
def headers(self):
return dict(self._headers.items())
@property
def kpis(self):
"""
Key Performance Indicators, holding things like number of returned frames/rois, etc.
:return:
"""
return self._kpis
@property
def trx(self):
return self.get_header(self._transaction_headers, self.id)
@trx.setter
def trx(self, value):
self.set_header(self._transaction_headers, value)
@property
def client(self):
return self.get_header(self._client_headers)
@property
def worker(self):
return self.get_worker(default="<unknown>")
def get_worker(self, default=None):
return self.get_header(self._worker_headers, default)
@property
def authorization(self):
""" Call authorization data used to authenticate the call """
return self.get_header(self.HEADER_AUTHORIZATION)
@property
def result(self):
return self._result
@property
def exec_async(self):
return self.get_header(self._async_headers) is not None
@exec_async.setter
def exec_async(self, value):
if value:
self.set_header(self._async_headers, "1")
else:
self.clear_header(self._async_headers)
@property
def host(self):
return self._host
@property
def auth_cookie(self):
return self._auth_cookie
@property
def json_flags(self):
return self._json_flags
@property
def extra_meta_fields(self):
return {}
def mark_end(self):
self._end_ts = time.time()
self._duration = int((self._end_ts - self._start_ts) * 1000)
def get_response(self,
include_stack: bool = None
) -> Tuple[Union[dict, str], str]:
"""
Get the response for this call.
:param include_stack: If True, stack trace stored in this call's result should
be included in the response (default follows configuration)
:return: Response data (encoded according to self.content_type) and the data's content type
"""
include_stack = (include_stack if include_stack is not None else
config.get("apiserver.return_stack_to_caller", False))
def make_version_number(
version: PartialVersion) -> Union[None, float, str]:
"""
Client versions <=2.0 expect expect endpoint versions in float format, otherwise throwing an exception
"""
if version is None:
return None
if self.requested_endpoint_version < PartialVersion("2.1"):
return float(str(version))
return str(version)
if self.result.raw_data and not self.failed:
# endpoint returned raw data and no error was detected, return raw data, no fancy dicts
return self.result.raw_data, self.result.content_type
else:
res = {
"meta": {
"id": self.id,
"trx": self.trx,
"endpoint": {
"name":
self.endpoint_name,
"requested_version":
make_version_number(self.requested_endpoint_version),
"actual_version":
make_version_number(self.actual_endpoint_version),
},
"result_code": self.result.code,
"result_subcode": self.result.subcode,
"result_msg": self.result.msg,
"error_stack":
self.result.traceback if include_stack else None,
"error_data": self.result.error_data,
**self.extra_meta_fields,
},
"data": self.result.data,
}
if self.content_type.lower() == JSON_CONTENT_TYPE:
try:
func = json.dumps if self._json_flags.pop(
"ensure_ascii", True) else json.dumps_notascii
res = func(res, **(self._json_flags or {}))
except Exception as ex:
# JSON serialization may fail, probably problem with data or error_data so pop it and try again
if not (self.result.data or self.result.error_data):
raise
self.result.data = None
self.result.error_data = None
msg = "Error serializing response data: " + str(ex)
self.set_error_result(code=500,
subcode=0,
msg=msg,
include_stack=False)
return self.get_response()
return res, self.content_type
def set_error_result(self,
msg,
code=500,
subcode=0,
include_stack=False,
error_data=None):
tb = format_exc() if include_stack else None
self._result = APICallResult(
data=self._result.data,
code=code,
subcode=subcode,
msg=msg,
traceback=tb,
error_data=error_data,
cookies=self._result.cookies,
)
def get_redacted_headers(self):
headers = self.headers.copy()
if not self.requires_authorization or self.auth:
# We won't log the authorization header if call shouldn't be authorized, or if it was successfully
# authorized. This means we'll only log authorization header for calls that failed to authorize (hopefully
# this will allow us to debug authorization errors).
for header, sep in self._redacted_headers.items():
if header in headers:
prefix, _, redact = headers[header].partition(sep)
headers[
header] = prefix + sep + f"<{len(redact)} bytes redacted>"
return headers
