def save_test(self) -> None:
"""Save a previously run test_id."""
try:
# touch the save file so it isn't deleted.
os.utime(
os.path.join(self.config["save_dir"], self.test_id),
(
thor.time(),
thor.time() +
(int(self.config["save_days"]) * 24 * 60 * 60),
),
)
location = b"?id=%s" % self.test_id.encode("ascii")
if self.descend:
location = b"%s&descend=True" % location
self.response_start(b"303", b"See Other",
[(b"Location", location)])
self.response_body("Redirecting to the saved test page...".encode(
self.config["charset"]))
except (OSError, IOError):
self.response_start(
b"500",
b"Internal Server Error",
[(b"Content-Type",
b"text/html; charset=%s" % self.charset_bytes)],
)
self.response_body(self.show_error("Sorry, I couldn't save that."))
self.response_done([])
def save_test(self):
"""Save a previously run test_id."""
try:
# touch the save file so it isn't deleted.
os.utime(
os.path.join(save_dir, self.test_id),
(
thor.time(),
thor.time() + (save_days * 24 * 60 * 60)
)
)
location = "?id=%s" % self.test_id
if self.descend:
location = "%s&descend=True" % location
self.response_start(
"303", "See Other", [
("Location", location)
])
self.response_body("Redirecting to the saved test page...")
except (OSError, IOError):
self.response_start(
"500", "Internal Server Error", [
("Content-Type", "text/html; charset=%s" % charset),
])
# TODO: better error message (through formatter?)
self.response_body(
error_template % "Sorry, I couldn't save that."
)
self.response_done([])
def extend_saved_test(webui: "RedWebUi") -> None:
"""Extend the expiry time of a previously run test_id."""
try:
# touch the save file so it isn't deleted.
os.utime(
os.path.join(webui.config["save_dir"], webui.test_id),
(
thor.time(),
thor.time() + (int(webui.config["save_days"]) * 24 * 60 * 60),
),
)
location = b"?id=%s" % webui.test_id.encode("ascii")
if webui.descend:
location = b"%s&descend=True" % location
webui.exchange.response_start(b"303", b"See Other",
[(b"Location", location)])
webui.output("Redirecting to the saved test page...")
except OSError:
webui.exchange.response_start(
b"500",
b"Internal Server Error",
[(b"Content-Type", b"text/html; charset=%s" % webui.charset_bytes)
],
)
webui.output("Sorry, I couldn't save that.")
webui.exchange.response_done([])
def write(self, content: bytes, lifetime: int) -> None:
"""
Write content to the file, marking it fresh for lifetime seconds.
Discard errors silently.
"""
try:
fd = gzip.open(self.path, 'w')
fd.write(content)
os.utime(self.path, (thor.time(), thor.time() + lifetime))
except (OSError, IOError, zlib.error):
return
finally:
if 'fd' in locals():
fd.close()
def write(self, content: bytes, lifetime: int) -> None:
"""
Write content to the file, marking it fresh for lifetime seconds.
Discard errors silently.
"""
try:
fd = gzip.open(self.path, "w")
fd.write(content)
os.utime(self.path, (thor.time(), thor.time() + lifetime))
except (OSError, IOError, zlib.error):
return
finally:
if "fd" in locals():
fd.close()
def __init__(self, *args: Any, **kw: Any) -> None:
Formatter.__init__(self, *args, **kw)
self.templates = Environment(
loader=PackageLoader("redbot.formatter"),
trim_blocks=True,
autoescape=select_autoescape(
enabled_extensions=("html", "xml"), default_for_string=True,
),
)
self.templates.filters.update(
{
"f_num": f_num,
"relative_time": relative_time,
"redbot_link": self.redbot_link,
}
)
self.templates.globals.update(
{
"formatter": self,
"version": __version__,
"baseuri": self.config["ui_uri"],
"static": self.config["static_root"],
"hcaptcha": self.config.get("hcaptcha_sitekey", "") != ""
and self.config.get("hcaptcha_secret", "") != "",
}
)
self.start = thor.time()
def _response_done(self, trailers):
"Finish anaylsing the response, handling any parse errors."
self._st.append('_response_done()')
state = self.state
state.res_complete = True
state.res_done_ts = thor.time()
state.transfer_length = self.exchange.input_transfer_length
state.header_length = self.exchange.input_header_length
# TODO: check trailers
if self.status_cb and state.type:
self.status_cb("fetched %s (%s)" % (state.uri, state.type))
state.res_body_md5 = self._md5_processor.digest()
state.res_body_post_md5 = self._md5_post_processor.digest()
checkCaching(state)
if state.method not in ['HEAD'] and state.res_status not in ['304']:
# check payload basics
if state.parsed_hdrs.has_key('content-length'):
if state.res_body_len == state.parsed_hdrs['content-length']:
state.set_message('header-content-length', rs.CL_CORRECT)
else:
state.set_message('header-content-length',
rs.CL_INCORRECT,
body_length=f_num(state.res_body_len)
)
if state.parsed_hdrs.has_key('content-md5'):
c_md5_calc = base64.encodestring(state.res_body_md5)[:-1]
if state.parsed_hdrs['content-md5'] == c_md5_calc:
state.set_message('header-content-md5', rs.CMD5_CORRECT)
else:
state.set_message('header-content-md5', rs.CMD5_INCORRECT,
calc_md5=c_md5_calc)
self.done()
self.finish_task()
def read(self):
"""
Read the file, returning its contents. If it does not exist or
cannot be read, returns None.
"""
if not path.exists(self.path):
return None
try:
fd = gzip.open(self.path)
except (OSError, IOError, zlib.error):
self.delete()
return None
try:
mtime = os.fstat(fd.fileno()).st_mtime
is_fresh = mtime > thor.time()
if not is_fresh:
self.delete()
return None
content = fd.read()
except IOError:
self.delete()
return None
finally:
fd.close()
return content
def body_done(self,
complete: bool,
trailers: RawHeaderListType = None) -> None:
"""
Signal that the body is done. Complete should be True if we
know it's complete (e.g., final chunk, Content-Length).
"""
self.complete = complete
self.complete_time = thor.time()
self.trailers = trailers or []
self.payload_md5 = self._md5_processor.digest()
self.decoded_md5 = self._md5_post_processor.digest()
if self.is_request or \
(not self.is_head_response and self.status_code not in ['304']):
# check payload basics
if 'content-length' in self.parsed_headers:
if self.payload_len == self.parsed_headers['content-length']:
self.add_note('header-content-length', CL_CORRECT)
else:
self.add_note('header-content-length',
CL_INCORRECT,
body_length=f_num(self.payload_len))
if 'content-md5' in self.parsed_headers:
c_md5_calc = base64.encodebytes(self.payload_md5)[:-1]
if self.parsed_headers['content-md5'] == c_md5_calc:
self.add_note('header-content-md5', CMD5_CORRECT)
else:
self.add_note('header-content-md5',
CMD5_INCORRECT,
calc_md5=c_md5_calc)
self.emit('content_available')
def run(self, done_cb=None):
"""
Make an asynchronous HTTP request to uri, calling status_cb as it's
updated and done_cb when it's done. Reason is used to explain what the
request is in the status callback.
"""
self.outstanding_tasks += 1
self._st.append('run(%s)' % str(done_cb))
self.done_cb = done_cb
state = self.state
if not self.preflight() or state.uri == None:
# generally a good sign that we're not going much further.
self.finish_task()
return
if 'user-agent' not in [i[0].lower() for i in state.req_hdrs]:
state.req_hdrs.append(
(u"User-Agent", u"RED/%s (http://redbot.org/)" % __version__))
self.exchange = self.client.exchange()
self.exchange.on('response_start', self._response_start)
self.exchange.on('response_body', self._response_body)
self.exchange.on('response_done', self._response_done)
self.exchange.on('error', self._response_error)
if self.status_cb and state.type:
self.status_cb("fetching %s (%s)" % (state.uri, state.type))
req_hdrs = [
(k.encode('ascii', 'replace'), v.encode('latin-1', 'replace')) \
for (k, v) in state.req_hdrs
]
self.exchange.request_start(state.method, state.uri, req_hdrs)
state.req_ts = thor.time()
if state.req_body != None:
self.exchange.request_body(state.req_body)
self.exchange.request_done([])
def final_status(self):
# See issue #51
# self.status("RED made %(reqs)s requests in %(elapse)2.3f seconds." % {
# 'reqs': fetch.total_requests,
self.status("RED finished in %(elapse)2.3f seconds." % {
'elapse': thor.time() - self.start
})
def run_continue(self, allowed: bool) -> None:
"""
Continue after getting the robots file.
"""
if not allowed:
self.response.http_error = RobotsTxtError()
self._fetch_done()
return
self.fetch_started = True
if 'user-agent' not in [i[0].lower() for i in self.request.headers]:
self.request.headers.append(("User-Agent", UA_STRING))
self.exchange = self.client.exchange()
self.exchange.on('response_nonfinal', self._response_nonfinal)
self.exchange.once('response_start', self._response_start)
self.exchange.on('response_body', self._response_body)
self.exchange.once('response_done', self._response_done)
self.exchange.on('error', self._response_error)
self.emit("status", "fetching %s (%s)" % (self.request.uri, self.check_name))
req_hdrs = [(k.encode('ascii', 'replace'), v.encode('ascii', 'replace'))
for (k, v) in self.request.headers] # FIXME: should complain
self.exchange.request_start(
self.request.method.encode('ascii'), self.request.uri.encode('ascii'), req_hdrs)
self.request.start_time = thor.time()
if self.request.payload != None:
self.exchange.request_body(self.request.payload)
self.transfer_out += len(self.request.payload)
self.exchange.request_done([])
def load_saved_test(self) -> None:
"""Load a saved test by test_id."""
try:
fd = gzip.open(
os.path.join(self.config.save_dir,
os.path.basename(self.test_id)))
mtime = os.fstat(fd.fileno()).st_mtime
except (OSError, IOError, TypeError, zlib.error):
self.response_start(
b"404", b"Not Found",
[(b"Content-Type",
b"text/html; charset=%s" % self.charset_bytes),
(b"Cache-Control", b"max-age=600, must-revalidate")])
self.response_body(
self.show_error(
"I'm sorry, I can't find that saved response."))
self.response_done([])
return
is_saved = mtime > thor.time()
try:
top_resource = pickle.load(fd)
except (pickle.PickleError, IOError, EOFError):
self.response_start(
b"500", b"Internal Server Error",
[(b"Content-Type",
b"text/html; charset=%s" % self.charset_bytes),
(b"Cache-Control", b"max-age=600, must-revalidate")])
self.response_body(
self.show_error("I'm sorry, I had a problem loading that."))
self.response_done([])
return
finally:
fd.close()
if self.check_name:
display_resource = top_resource.subreqs.get(
self.check_name, top_resource)
else:
display_resource = top_resource
formatter = find_formatter(self.format, 'html', top_resource.descend)(
self.ui_uri,
self.config.lang,
self.output,
allow_save=(not is_saved),
is_saved=True,
test_id=self.test_id)
content_type = "%s; charset=%s" % (formatter.media_type,
self.config.charset)
self.response_start(
b"200", b"OK",
[(b"Content-Type", content_type.encode('ascii')),
(b"Cache-Control", b"max-age=3600, must-revalidate")])
@thor.events.on(formatter)
def formatter_done() -> None:
self.response_done([])
formatter.bind_resource(display_resource)
def save_test(self) -> None:
"""Save a previously run test_id."""
try:
# touch the save file so it isn't deleted.
os.utime(os.path.join(self.config.save_dir, self.test_id), (
thor.time(), thor.time() + (self.config.save_days * 24 * 60 * 60)))
location = "?id=%s" % self.test_id
if self.descend:
location = "%s&descend=True" % location
self.response_start("303", "See Other", [("Location", location)])
self.response_body("Redirecting to the saved test page...".encode(self.config.charset))
except (OSError, IOError):
self.response_start(b"500", b"Internal Server Error",
[(b"Content-Type", b"text/html; charset=%s" % self.charset_bytes),])
self.response_body(self.show_error("Sorry, I couldn't save that."))
self.response_done([])
def _response_done(self, trailers):
"Finish analysing the response, handling any parse errors."
self._st.append('_response_done()')
state = self.state
state.res_complete = True
state.res_done_ts = thor.time()
state.transfer_length = self.exchange.input_transfer_length
state.header_length = self.exchange.input_header_length
# TODO: check trailers
if self.status_cb and state.type:
self.status_cb("fetched %s (%s)" % (state.uri, state.type))
state.res_body_md5 = self._md5_processor.digest()
state.res_body_post_md5 = self._md5_post_processor.digest()
checkCaching(state)
if state.method not in ['HEAD'] and state.res_status not in ['304']:
# check payload basics
if state.parsed_hdrs.has_key('content-length'):
if state.res_body_len == state.parsed_hdrs['content-length']:
state.set_message('header-content-length', rs.CL_CORRECT)
else:
state.set_message('header-content-length',
rs.CL_INCORRECT,
body_length=f_num(state.res_body_len))
if state.parsed_hdrs.has_key('content-md5'):
c_md5_calc = base64.encodestring(state.res_body_md5)[:-1]
if state.parsed_hdrs['content-md5'] == c_md5_calc:
state.set_message('header-content-md5', rs.CMD5_CORRECT)
else:
state.set_message('header-content-md5',
rs.CMD5_INCORRECT,
calc_md5=c_md5_calc)
self.done()
self.finish_task()
def run(self, done_cb=None):
"""
Make an asynchronous HTTP request to uri, calling status_cb as it's
updated and done_cb when it's done. Reason is used to explain what the
request is in the status callback.
"""
self.outstanding_tasks += 1
self._st.append('run(%s)' % str(done_cb))
self.done_cb = done_cb
state = self.state
if not self.preflight() or state.uri == None:
# generally a good sign that we're not going much further.
self.finish_task()
return
if 'user-agent' not in [i[0].lower() for i in state.req_hdrs]:
state.req_hdrs.append(
(u"User-Agent", u"RED/%s (http://redbot.org/)" % __version__))
self.exchange = self.client.exchange()
self.exchange.on('response_start', self._response_start)
self.exchange.on('response_body', self._response_body)
self.exchange.on('response_done', self._response_done)
self.exchange.on('error', self._response_error)
if self.status_cb and state.type:
self.status_cb("fetching %s (%s)" % (state.uri, state.type))
req_hdrs = [
(k.encode('ascii', 'replace'), v.encode('latin-1', 'replace')) \
for (k, v) in state.req_hdrs
]
self.exchange.request_start(state.method, state.uri, req_hdrs)
state.req_ts = thor.time()
if state.req_body != None:
self.exchange.request_body(state.req_body)
self.exchange.request_done([])
def body_done(self, complete: bool, trailers: RawHeaderListType = None) -> None:
"""
Signal that the body is done. Complete should be True if we
know it's complete (e.g., final chunk, Content-Length).
"""
self.complete = complete
self.complete_time = thor.time()
self.trailers = trailers or []
self.payload_md5 = self._md5_processor.digest()
self.decoded_md5 = self._md5_post_processor.digest()
if self.is_request or \
(not self.is_head_response and self.status_code not in ['304']):
# check payload basics
if 'content-length' in self.parsed_headers:
if self.payload_len == self.parsed_headers['content-length']:
self.add_note('header-content-length', CL_CORRECT)
else:
self.add_note('header-content-length',
CL_INCORRECT,
body_length=f_num(self.payload_len))
if 'content-md5' in self.parsed_headers:
c_md5_calc = base64.encodebytes(self.payload_md5)[:-1]
if self.parsed_headers['content-md5'] == c_md5_calc:
self.add_note('header-content-md5', CMD5_CORRECT)
else:
self.add_note('header-content-md5',
CMD5_INCORRECT, calc_md5=c_md5_calc)
self.emit('content_available')
def read(self) -> bytes:
"""
Read the file, returning its contents. If it does not exist or
cannot be read, returns None.
"""
if not path.exists(self.path):
return None
try:
fd = gzip.open(self.path)
except (OSError, IOError, zlib.error):
self.delete()
return None
try:
mtime = os.fstat(fd.fileno()).st_mtime
is_fresh = mtime > thor.time()
if not is_fresh:
self.delete()
return None
content = fd.read()
except IOError:
self.delete()
return None
finally:
if "fd" in locals():
fd.close()
return content
def run_continue(self, allowed: bool) -> None:
"""
Continue after getting the robots file.
"""
if not allowed:
self.response.http_error = RobotsTxtError()
self._fetch_done()
return
self.fetch_started = True
if 'user-agent' not in [i[0].lower() for i in self.request.headers]:
self.request.headers.append(("User-Agent", UA_STRING))
self.exchange = self.client.exchange()
self.exchange.on('response_nonfinal', self._response_nonfinal)
self.exchange.once('response_start', self._response_start)
self.exchange.on('response_body', self._response_body)
self.exchange.once('response_done', self._response_done)
self.exchange.on('error', self._response_error)
self.emit("status", "fetching %s (%s)" % (self.request.uri, self.check_name))
req_hdrs = [(k.encode('ascii'), v.encode('ascii')) for (k, v) in self.request.headers]
self.exchange.request_start(
self.request.method.encode('ascii'), self.request.uri.encode('ascii'), req_hdrs)
self.request.start_time = thor.time()
if self.request.payload != None:
self.exchange.request_body(self.request.payload)
self.transfer_out += len(self.request.payload)
self.exchange.request_done([])
def final_status(self) -> None:
# See issue #51
# self.status("REDbot made %(reqs)s requests in %(elapse)2.3f seconds." % {
# 'reqs': fetch.total_requests,
self.status("")
self.output("""
<div id="final_status">%(elapse)2.2f seconds</div>
""" % {'elapse': thor.time() - self.start})
def _response_start(self, status: bytes, phrase: bytes,
res_headers: RawHeaderListType) -> None:
"Process the response start-line and headers."
self.response.start_time = thor.time()
self.response.process_top_line(self.exchange.res_version, status, phrase)
self.response.process_raw_headers(res_headers)
StatusChecker(self.response, self.request)
checkCaching(self.response, self.request)
def _response_start(self, status: bytes, phrase: bytes,
res_headers: RawHeaderListType) -> None:
"Process the response start-line and headers."
self.response.start_time = thor.time()
self.response.process_top_line(self.exchange.res_version, status, phrase)
self.response.process_raw_headers(res_headers)
StatusChecker(self.response, self.request)
checkCaching(self.response, self.request)
def final_status(self) -> None:
# See issue #51
# self.status("REDbot made %(reqs)s requests in %(elapse)2.3f seconds." % {
# 'reqs': fetch.total_requests,
self.status("")
self.output("""
<div id="final_status">%(elapse)2.2f seconds</div>
""" % {'elapse': thor.time() - self.start})
def status(self, message):
"Update the status bar of the browser"
self.output(u"""
<script>
<!-- %3.3f
window.status="%s";
-->
</script>
""" % (thor.time() - self.start, e_html(message)))
def debug(self, message: str) -> None:
"Debug to console."
self.output("""
<script>
<!--
console.log("%3.3f %s");
-->
</script>
""" % (thor.time() - self.start, e_js(message)))
def status(self, message: str) -> None:
"Update the status bar of the browser"
self.output("""
<script>
<!-- %3.3f
$('#red_status').text("%s");
-->
</script>
""" % (thor.time() - self.start, e_html(message)))
def status(self, message: str) -> None:
"Update the status bar of the browser"
self.output("""
<script>
<!-- %3.3f
$('#red_status').text("%s");
-->
</script>
""" % (thor.time() - self.start, e_html(message)))
def save_test(self):
"""Save a previously run test_id."""
try:
# touch the save file so it isn't deleted.
os.utime(os.path.join(save_dir, self.test_id),
(thor.time(), thor.time() + (save_days * 24 * 60 * 60)))
location = "?id=%s" % self.test_id
if self.descend:
location = "%s&descend=True" % location
self.response_start("303", "See Other", [("Location", location)])
self.response_body("Redirecting to the saved test page...")
except (OSError, IOError):
self.response_start("500", "Internal Server Error", [
("Content-Type", "text/html; charset=%s" % charset),
])
# TODO: better error message (through formatter?)
self.response_body(error_template % "Sorry, I couldn't save that.")
self.response_done([])
def debug(self, message: str) -> None:
"Debug to console."
self.output("""
<script>
<!--
console.log("%3.3f %s");
-->
</script>
""" % (thor.time() - self.start, e_js(message)))
def load_saved_test(self):
"""Load a saved test by test_id."""
try:
fd = gzip.open(os.path.join(
save_dir, os.path.basename(self.test_id)
))
mtime = os.fstat(fd.fileno()).st_mtime
except (OSError, IOError, TypeError, zlib.error):
self.response_start(
"404", "Not Found", [
("Content-Type", "text/html; charset=%s" % charset),
("Cache-Control", "max-age=600, must-revalidate")
])
# TODO: better error page (through formatter?)
self.response_body(error_template %
"I'm sorry, I can't find that saved response."
)
self.response_done([])
return
is_saved = mtime > thor.time()
try:
state = pickle.load(fd)
except (pickle.PickleError, EOFError):
self.response_start(
"500", "Internal Server Error", [
("Content-Type", "text/html; charset=%s" % charset),
("Cache-Control", "max-age=600, must-revalidate")
])
# TODO: better error page (through formatter?)
self.response_body(error_template %
"I'm sorry, I had a problem reading that response."
)
self.response_done([])
return
finally:
fd.close()
formatter = find_formatter(self.format, 'html', self.descend)(
self.base_uri, state.request.uri, state.orig_req_hdrs, lang,
self.output, allow_save=(not is_saved), is_saved=True,
test_id=self.test_id
)
self.response_start(
"200", "OK", [
("Content-Type", "%s; charset=%s" % (
formatter.media_type, charset)),
("Cache-Control", "max-age=3600, must-revalidate")
])
if self.check_type:
# TODO: catch errors
state = state.subreqs.get(self.check_type, None)
formatter.start_output()
formatter.set_state(state)
formatter.finish_output()
self.response_done([])
def _response_start(self, status, phrase, res_headers):
"Process the response start-line and headers."
self._st.append('_response_start(%s, %s)' % (status, phrase))
self.response.start_time = thor.time()
self.response.version = self.exchange.res_version
self.response.status_code = status.decode('iso-8859-1', 'replace')
self.response.status_phrase = phrase.decode('iso-8859-1', 'replace')
self.response.set_headers(res_headers)
StatusChecker(self.response, self.request)
checkCaching(self.response, self.request)
def load_saved_test(self):
"""Load a saved test by test_id."""
try:
fd = gzip.open(
os.path.join(save_dir, os.path.basename(self.test_id)))
mtime = os.fstat(fd.fileno()).st_mtime
except (OSError, IOError, TypeError, zlib.error):
self.response_start(
"404", "Not Found",
[("Content-Type", "text/html; charset=%s" % charset),
("Cache-Control", "max-age=600, must-revalidate")])
# TODO: better error page (through formatter?)
self.response_body(error_template %
"I'm sorry, I can't find that saved response.")
self.response_done([])
return
is_saved = mtime > thor.time()
try:
state = pickle.load(fd)
except (pickle.PickleError, EOFError):
self.response_start(
"500", "Internal Server Error",
[("Content-Type", "text/html; charset=%s" % charset),
("Cache-Control", "max-age=600, must-revalidate")])
# TODO: better error page (through formatter?)
self.response_body(
error_template %
"I'm sorry, I had a problem reading that response.")
self.response_done([])
return
finally:
fd.close()
formatter = find_formatter(self.format, 'html',
self.descend)(self.base_uri,
state.uri,
state.orig_req_hdrs,
lang,
self.output,
allow_save=(not is_saved),
is_saved=True,
test_id=self.test_id)
self.response_start(
"200", "OK", [("Content-Type", "%s; charset=%s" %
(formatter.media_type, charset)),
("Cache-Control", "max-age=3600, must-revalidate")])
if self.req_type:
# TODO: catch errors
state = state.subreqs.get(self.req_type, None)
formatter.start_output()
formatter.set_red(state)
formatter.finish_output()
self.response_done([])
def verify_human(self, human_time: int, human_hmac: str) -> bool:
"""
Check the user's human HMAC.
"""
computed_hmac = hmac.new(self.secret, bytes(str(human_time), "ascii"),
"sha512")
is_valid = human_hmac == computed_hmac.hexdigest()
if is_valid and human_time >= thor.time():
return True
else:
return False
def _response_done(self, trailers):
"Finish analysing the response, handling any parse errors."
self._st.append('_response_done()')
self.response.complete_time = thor.time()
self.response.transfer_length = self.exchange.input_transfer_length
self.response.header_length = self.exchange.input_header_length
self.response.body_done(True, trailers)
if self.status_cb and self.name:
self.status_cb("fetched %s (%s)" % (
self.request.uri, self.name
))
self.done()
self.finish_task()
def _response_start(self, status, phrase, res_headers):
"Process the response start-line and headers."
state = self.state
state.res_ts = thor.time()
state.res_version = self.exchange.res_version
state.res_status = status.decode('iso-8859-1', 'replace')
state.res_phrase = phrase.decode('iso-8859-1', 'replace')
state.res_hdrs = res_headers
redbot.headers.process_headers(state)
redbot.status_check.ResponseStatusChecker(state)
state.res_body_enc = state.parsed_hdrs.get(
'content-type', (None, {})
)[1].get('charset', 'utf-8') # default isn't UTF-8, but oh well
def robot(results: Tuple[str, bool]) -> None:
url, robot_ok = results
if robot_ok:
self.continue_test(top_resource, formatter)
else:
valid_till = str(int(thor.time()) + 60)
robot_hmac = hmac.new(self._robot_secret, bytes(valid_till, 'ascii'))
self.response_start(b"403", b"Forbidden", [
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"no-cache")])
formatter.start_output()
formatter.error_output("This site doesn't allow robots. If you are human, please <a href='?uri=%s&robot_time=%s&robot_hmac=%s'>click here</a>." % (self.test_uri, valid_till, robot_hmac.hexdigest()) )
self.response_done([])
def _response_start(self, status, phrase, res_headers):
"Process the response start-line and headers."
state = self.state
state.res_ts = thor.time()
state.res_version = self.exchange.res_version
state.res_status = status.decode('iso-8859-1', 'replace')
state.res_phrase = phrase.decode('iso-8859-1', 'replace')
state.res_hdrs = res_headers
redbot.headers.process_headers(state)
redbot.status_check.ResponseStatusChecker(state)
state.res_body_enc = state.parsed_hdrs.get(
'content-type',
(None, {}))[1].get('charset',
'utf-8') # default isn't UTF-8, but oh well
def _response_error(self, error):
state = self.state
state.res_done_ts = thor.time()
state.res_error = error
if isinstance(error, httperr.BodyForbiddenError):
state.set_message('header-none', rs.BODY_NOT_ALLOWED)
# elif isinstance(error, httperr.ExtraDataErr):
# state.res_body_len += len(err.get('detail', ''))
elif isinstance(error, httperr.ChunkError):
err_msg = error.detail[:20] or ""
state.set_message('header-transfer-encoding', rs.BAD_CHUNK,
chunk_sample=e(err_msg.encode('string_escape'))
)
self.done()
self.finish_task()
def _response_error(self, error):
"Handle an error encountered while fetching the response."
self._st.append('_response_error(%s)' % (str(error)))
self.response.complete_time = thor.time()
self.response.http_error = error
if isinstance(error, httperr.BodyForbiddenError):
self.add_note('header-none', rs.BODY_NOT_ALLOWED)
# elif isinstance(error, httperr.ExtraDataErr):
# res.payload_len += len(err.get('detail', ''))
elif isinstance(error, httperr.ChunkError):
err_msg = error.detail[:20] or ""
self.add_note('header-transfer-encoding', rs.BAD_CHUNK,
chunk_sample=err_msg.encode('string_escape')
)
self.done()
self.finish_task()
def _response_error(self, error):
state = self.state
state.res_done_ts = thor.time()
state.res_error = error
if isinstance(error, httperr.BodyForbiddenError):
state.set_message('header-none', rs.BODY_NOT_ALLOWED)
# elif isinstance(error, httperr.ExtraDataErr):
# state.res_body_len += len(err.get('detail', ''))
elif isinstance(error, httperr.ChunkError):
err_msg = error.detail[:20] or ""
state.set_message('header-transfer-encoding',
rs.BAD_CHUNK,
chunk_sample=e(err_msg.encode('string_escape')))
self.done()
self.finish_task()
def issue_human(self) -> RawHeaderListType:
"""
Return cookie headers for later verification that this is a human.
"""
human_time = str(int(thor.time()) + self.token_lifetime)
human_hmac = hmac.new(self.secret, bytes(human_time, "ascii"),
"sha512").hexdigest()
return [
(
b"Set-Cookie",
f"human_time={human_time}; Max-Age={self.token_lifetime}; SameSite=Strict"
.encode("ascii"),
),
(
b"Set-Cookie",
f"human_hmac={human_hmac}; Max-Age={self.token_lifetime}; SameSite=Strict"
.encode("ascii"),
),
]
def verify_slack_secret(webui: "RedWebUi") -> bool:
"""Verify the slack secret."""
slack_signing_secret = webui.config.get("slack_signing_secret",
fallback="").encode("utf-8")
timestamp = get_header(webui.req_headers, b"x-slack-request-timestamp")
if not timestamp or not timestamp[0].isdigit():
return False
timestamp = timestamp[0]
if abs(thor.time() - int(timestamp)) > 60 * 5:
return False
sig_basestring = b"v0:" + timestamp + b":" + webui.req_body
signature = (
f"v0={hmac.new(slack_signing_secret, sig_basestring, 'sha256').hexdigest()}"
)
presented_signature = get_header(webui.req_headers, b"x-slack-signature")
if not presented_signature:
return False
presented_sig = presented_signature[0].decode("utf-8")
return hmac.compare_digest(signature, presented_sig)
def load_saved_test(self) -> None:
"""Load a saved test by test_id."""
try:
fd = gzip.open(os.path.join(self.config.save_dir, os.path.basename(self.test_id)))
mtime = os.fstat(fd.fileno()).st_mtime
except (OSError, IOError, TypeError, zlib.error):
self.response_start(b"404", b"Not Found", [
(b"Content-Type", b"text/html; charset=%s" % self.charset_bytes),
(b"Cache-Control", b"max-age=600, must-revalidate")])
self.response_body(self.show_error("I'm sorry, I can't find that saved response."))
self.response_done([])
return
is_saved = mtime > thor.time()
try:
top_resource = pickle.load(fd)
except (pickle.PickleError, IOError, EOFError):
self.response_start(b"500", b"Internal Server Error", [
(b"Content-Type", b"text/html; charset=%s" % self.charset_bytes),
(b"Cache-Control", b"max-age=600, must-revalidate")])
self.response_body(self.show_error("I'm sorry, I had a problem loading that."))
self.response_done([])
return
finally:
fd.close()
if self.check_name:
display_resource = top_resource.subreqs.get(self.check_name, top_resource)
else:
display_resource = top_resource
formatter = find_formatter(self.format, 'html', top_resource.descend)(
self.ui_uri, self.config.lang, self.output,
allow_save=(not is_saved), is_saved=True, test_id=self.test_id)
content_type = "%s; charset=%s" % (formatter.media_type, self.config.charset)
self.response_start(b"200", b"OK", [
(b"Content-Type", content_type.encode('ascii')),
(b"Cache-Control", b"max-age=3600, must-revalidate")])
@thor.events.on(formatter)
def formatter_done() -> None:
self.response_done([])
formatter.bind_resource(display_resource)
def robot(results: Tuple[str, bool]) -> None:
url, robot_ok = results
if robot_ok:
self.continue_test(top_resource, formatter)
else:
valid_till = str(int(thor.time()) + 60)
robot_hmac = hmac.new(self._robot_secret,
bytes(valid_till, "ascii"))
self.response_start(
b"403",
b"Forbidden",
[
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"no-cache"),
],
)
formatter.start_output()
formatter.error_output(
"This site doesn't allow robots. If you are human, please <a href='?uri=%s&robot_time=%s&robot_hmac=%s'>click here</a>."
% (self.test_uri, valid_till, robot_hmac.hexdigest()))
self.response_done([])
def check(self) -> None:
"""
Make an asynchronous HTTP request to uri, emitting 'status' as it's
updated and 'fetch_done' when it's done. Reason is used to explain what the
request is in the status callback.
"""
if not self.preflight() or self.request.uri is None:
# generally a good sign that we're not going much further.
self._fetch_done()
return
self.fetch_started = True
if "user-agent" not in [i[0].lower() for i in self.request.headers]:
self.request.headers.append(("User-Agent", UA_STRING))
self.exchange = self.client.exchange()
self.exchange.on("response_nonfinal", self._response_nonfinal)
self.exchange.once("response_start", self._response_start)
self.exchange.on("response_body", self._response_body)
self.exchange.once("response_done", self._response_done)
self.exchange.on("error", self._response_error)
self.emit("status",
"fetching %s (%s)" % (self.request.uri, self.check_name))
self.emit("debug",
"fetching %s (%s)" % (self.request.uri, self.check_name))
req_hdrs = [(k.encode("ascii",
"replace"), v.encode("ascii", "replace"))
for (k, v) in self.request.headers]
self.exchange.request_start(
self.request.method.encode("ascii"),
self.request.uri.encode("ascii"),
req_hdrs,
)
self.request.start_time = thor.time()
if not self.fetch_done: # the request could have immediately failed.
if self.request.payload is not None:
self.exchange.request_body(self.request.payload)
self.transfer_out += len(self.request.payload)
if not self.fetch_done: # the request could have immediately failed.
self.exchange.request_done([])
def run_continue(self, robots_txt):
"""
Continue after getting the robots file.
TODO: refactor callback style into events.
"""
if robots_txt == "": # empty or non-200
pass
else:
checker = RobotFileParser()
checker.parse(robots_txt.decode('ascii', 'replace').encode('ascii', 'replace').splitlines())
if not checker.can_fetch(UA_STRING, self.request.uri):
self.response.http_error = RobotsTxtError()
self.finish_task()
return # TODO: show error?
if 'user-agent' not in [i[0].lower() for i in self.request.headers]:
self.request.headers.append(
(u"User-Agent", UA_STRING))
self.exchange = self.client.exchange()
self.exchange.on('response_start', self._response_start)
self.exchange.on('response_body', self._response_body)
self.exchange.on('response_done', self._response_done)
self.exchange.on('error', self._response_error)
if self.status_cb and self.name:
self.status_cb("fetching %s (%s)" % (
self.request.uri, self.name
))
req_hdrs = [
(k.encode('ascii', 'replace'), v.encode('latin-1', 'replace')) \
for (k, v) in self.request.headers
]
self.exchange.request_start(
self.request.method, self.request.uri, req_hdrs
)
self.request.start_time = thor.time()
if self.request.payload != None:
self.exchange.request_body(self.request.payload)
self.transfer_out += len(self.request.payload)
self.exchange.request_done([])
def run_continue(self, allowed: bool) -> None:
"""
Continue after getting the robots file.
"""
if not allowed:
self.response.http_error = RobotsTxtError()
self._fetch_done()
return
self.fetch_started = True
if "user-agent" not in [i[0].lower() for i in self.request.headers]:
self.request.headers.append(("User-Agent", UA_STRING))
self.exchange = self.client.exchange()
self.exchange.on("response_nonfinal", self._response_nonfinal)
self.exchange.once("response_start", self._response_start)
self.exchange.on("response_body", self._response_body)
self.exchange.once("response_done", self._response_done)
self.exchange.on("error", self._response_error)
self.emit("status",
"fetching %s (%s)" % (self.request.uri, self.check_name))
self.emit("debug",
"fetching %s (%s)" % (self.request.uri, self.check_name))
req_hdrs = [
(k.encode("ascii", "replace"), v.encode("ascii", "replace"))
for (k, v) in self.request.headers
] # FIXME: should complain
self.exchange.request_start(
self.request.method.encode("ascii"),
self.request.uri.encode("ascii"),
req_hdrs,
)
self.request.start_time = thor.time()
if not self.fetch_done: # the request could have immediately failed.
if self.request.payload is not None:
self.exchange.request_body(self.request.payload)
self.transfer_out += len(self.request.payload)
if not self.fetch_done: # the request could have immediately failed.
self.exchange.request_done([])
def run_test(self) -> None:
"""Test a URI."""
# try to initialise stored test results
if self.config.get("save_dir", "") and os.path.exists(
self.config["save_dir"]):
try:
fd, self.save_path = tempfile.mkstemp(
prefix="", dir=self.config["save_dir"])
self.test_id = os.path.split(self.save_path)[1]
except (OSError, IOError):
# Don't try to store it.
self.test_id = None # should already be None, but make sure
top_resource = HttpResource(self.config, descend=self.descend)
self.timeout = thor.schedule(
int(self.config["max_runtime"]),
self.timeoutError,
top_resource.show_task_map,
)
top_resource.set_request(self.test_uri, req_hdrs=self.req_hdrs)
formatter = find_formatter(self.format, "html", self.descend)(
self.config,
self.output,
allow_save=self.test_id,
is_saved=False,
test_id=self.test_id,
descend=self.descend,
)
# referer limiting
referers = []
for hdr, value in self.req_hdrs:
if hdr.lower() == "referer":
referers.append(value)
referer_error = None
if len(referers) > 1:
referer_error = "Multiple referers not allowed."
if referers and urlsplit(
referers[0]).hostname in self.referer_spam_domains:
referer_error = "Referer not allowed."
if referer_error:
self.response_start(
b"403",
b"Forbidden",
[
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=360, must-revalidate"),
],
)
formatter.start_output()
formatter.error_output(referer_error)
self.response_done([])
return
# robot human check
if self.robot_time and self.robot_time.isdigit() and self.robot_hmac:
valid_till = int(self.robot_time)
computed_hmac = hmac.new(self._robot_secret,
bytes(self.robot_time, "ascii"))
is_valid = self.robot_hmac == computed_hmac.hexdigest()
if is_valid and valid_till >= thor.time():
self.continue_test(top_resource, formatter)
return
else:
self.response_start(
b"403",
b"Forbidden",
[
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=60, must-revalidate"),
],
)
formatter.start_output()
formatter.error_output("Naughty.")
self.response_done([])
self.error_log("Naughty robot key.")
# enforce client limits
if self.config.getint("limit_client_tests", fallback=0):
client_id = self.get_client_id()
if client_id:
if self._client_counts.get(
client_id,
0) > self.config.getint("limit_client_tests"):
self.response_start(
b"429",
b"Too Many Requests",
[
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=60, must-revalidate"),
],
)
formatter.start_output()
formatter.error_output(
"Your client is over limit. Please try later.")
self.response_done([])
self.error_log("client over limit: %s" %
client_id.decode("idna"))
return
self._client_counts[client_id] += 1
# enforce origin limits
if self.config.getint("limit_origin_tests", fallback=0):
origin = url_to_origin(self.test_uri)
if origin:
if self._origin_counts.get(
origin, 0) > self.config.getint("limit_origin_tests"):
self.response_start(
b"429",
b"Too Many Requests",
[
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=60, must-revalidate"),
],
)
formatter.start_output()
formatter.error_output(
"Origin is over limit. Please try later.")
self.response_done([])
self.error_log("origin over limit: %s" % origin)
return
self._origin_counts[origin] += 1
# check robots.txt
robot_fetcher = RobotFetcher(self.config)
@thor.events.on(robot_fetcher)
def robot(results: Tuple[str, bool]) -> None:
url, robot_ok = results
if robot_ok:
self.continue_test(top_resource, formatter)
else:
valid_till = str(int(thor.time()) + 60)
robot_hmac = hmac.new(self._robot_secret,
bytes(valid_till, "ascii"))
self.response_start(
b"403",
b"Forbidden",
[
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"no-cache"),
],
)
formatter.start_output()
formatter.error_output(
"This site doesn't allow robots. If you are human, please <a href='?uri=%s&robot_time=%s&robot_hmac=%s'>click here</a>."
% (self.test_uri, valid_till, robot_hmac.hexdigest()))
self.response_done([])
robot_fetcher.check_robots(HttpRequest.iri_to_uri(self.test_uri))
def __init__(self, *args: Any, **kw: Any) -> None:
Formatter.__init__(self, *args, **kw)
self.hidden_text = [] # type: List[Tuple[str, str]]
self.start = thor.time()
def load_saved_test(webui: "RedWebUi") -> None:
"""Load a saved test by test_id."""
try:
with gzip.open(
os.path.join(webui.config["save_dir"],
os.path.basename(webui.test_id))) as fd:
mtime = os.fstat(fd.fileno()).st_mtime
is_saved = mtime > thor.time()
top_resource = pickle.load(fd)
except (OSError, TypeError):
webui.exchange.response_start(
b"404",
b"Not Found",
[
(b"Content-Type",
b"text/html; charset=%s" % webui.charset_bytes),
(b"Cache-Control", b"max-age=600, must-revalidate"),
],
)
webui.output("I'm sorry, I can't find that saved response.")
webui.exchange.response_done([])
return
except (pickle.PickleError, zlib.error, EOFError):
webui.exchange.response_start(
b"500",
b"Internal Server Error",
[
(b"Content-Type",
b"text/html; charset=%s" % webui.charset_bytes),
(b"Cache-Control", b"max-age=600, must-revalidate"),
],
)
webui.output("I'm sorry, I had a problem loading that.")
webui.exchange.response_done([])
return
if webui.check_name:
display_resource = top_resource.subreqs.get(webui.check_name,
top_resource)
else:
display_resource = top_resource
formatter = find_formatter(webui.format, "html", top_resource.descend)(
webui.config,
display_resource,
webui.output,
allow_save=(not is_saved),
is_saved=True,
test_id=webui.test_id,
)
webui.exchange.response_start(
b"200",
b"OK",
[
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=3600, must-revalidate"),
],
)
@thor.events.on(formatter)
def formatter_done() -> None:
webui.exchange.response_done([])
formatter.bind_resource(display_resource)
def run_test(self) -> None:
"""Test a URI."""
# try to initialise stored test results
if self.config.get('save_dir', "") and os.path.exists(self.config['save_dir']):
try:
fd, self.save_path = tempfile.mkstemp(prefix='', dir=self.config['save_dir'])
self.test_id = os.path.split(self.save_path)[1]
except (OSError, IOError):
# Don't try to store it.
self.test_id = None # should already be None, but make sure
top_resource = HttpResource(self.config, descend=self.descend)
self.timeout = thor.schedule(int(self.config['max_runtime']), self.timeoutError,
top_resource.show_task_map)
top_resource.set_request(self.test_uri, req_hdrs=self.req_hdrs)
formatter = find_formatter(self.format, 'html', self.descend)(
self.config, self.output, allow_save=self.test_id, is_saved=False,
test_id=self.test_id, descend=self.descend)
# referer limiting
referers = []
for hdr, value in self.req_hdrs:
if hdr.lower() == 'referer':
referers.append(value)
referer_error = None
if len(referers) > 1:
referer_error = "Multiple referers not allowed."
if referers and urlsplit(referers[0]).hostname in self.referer_spam_domains:
referer_error = "Referer not allowed."
if referer_error:
self.response_start(b"403", b"Forbidden", [
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=360, must-revalidate")])
formatter.start_output()
formatter.error_output(referer_error)
self.response_done([])
return
# robot human check
if self.robot_time and self.robot_time.isdigit() and self.robot_hmac:
valid_till = int(self.robot_time)
computed_hmac = hmac.new(self._robot_secret, bytes(self.robot_time, 'ascii'))
is_valid = self.robot_hmac == computed_hmac.hexdigest()
if is_valid and valid_till >= thor.time():
self.continue_test(top_resource, formatter)
return
else:
self.response_start(b"403", b"Forbidden", [
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=60, must-revalidate")])
formatter.start_output()
formatter.error_output("Naughty.")
self.response_done([])
self.error_log("Naughty robot key.")
# enforce client limits
if self.config.getint('limit_client_tests', fallback=0):
client_id = self.get_client_id()
if client_id:
if self._client_counts.get(client_id, 0) > \
self.config.getint('limit_client_tests'):
self.response_start(b"429", b"Too Many Requests", [
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=60, must-revalidate")])
formatter.start_output()
formatter.error_output("Your client is over limit. Please try later.")
self.response_done([])
self.error_log("client over limit: %s" % client_id.decode('idna'))
return
self._client_counts[client_id] += 1
# enforce origin limits
if self.config.getint('limit_origin_tests', fallback=0):
origin = url_to_origin(self.test_uri)
if origin:
if self._origin_counts.get(origin, 0) > \
self.config.getint('limit_origin_tests'):
self.response_start(b"429", b"Too Many Requests", [
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"max-age=60, must-revalidate")])
formatter.start_output()
formatter.error_output("Origin is over limit. Please try later.")
self.response_done([])
self.error_log("origin over limit: %s" % origin)
return
self._origin_counts[origin] += 1
# check robots.txt
robot_fetcher = RobotFetcher(self.config)
@thor.events.on(robot_fetcher)
def robot(results: Tuple[str, bool]) -> None:
url, robot_ok = results
if robot_ok:
self.continue_test(top_resource, formatter)
else:
valid_till = str(int(thor.time()) + 60)
robot_hmac = hmac.new(self._robot_secret, bytes(valid_till, 'ascii'))
self.response_start(b"403", b"Forbidden", [
(b"Content-Type", formatter.content_type()),
(b"Cache-Control", b"no-cache")])
formatter.start_output()
formatter.error_output("This site doesn't allow robots. If you are human, please <a href='?uri=%s&robot_time=%s&robot_hmac=%s'>click here</a>." % (self.test_uri, valid_till, robot_hmac.hexdigest()) )
self.response_done([])
robot_fetcher.check_robots(HttpRequest.iri_to_uri(self.test_uri))
def __init__(self, *args: Any, **kw: Any) -> None:
Formatter.__init__(self, *args, **kw)
self.hidden_text = [] # type: List[Tuple[str, str]]
self.start = thor.time()
def __init__(self, *args, **kw):
Formatter.__init__(self, *args, **kw)
self.hidden_text = []
self.start = thor.time()
