def continue_test(self, top_resource: HttpResource, formatter: Formatter) -> None: "Preliminary checks are done; actually run the test." @thor.events.on(formatter) def formatter_done() -> None: self.response_done([]) if self.test_id: try: tmp_file = gzip.open(self.save_path, 'w') pickle.dump(top_resource, tmp_file) tmp_file.close() except (IOError, zlib.error, pickle.PickleError): pass # we don't cry if we can't store it. # log excessive traffic ti = sum([i.transfer_in for i, t in top_resource.linked], top_resource.transfer_in) to = sum([i.transfer_out for i, t in top_resource.linked], top_resource.transfer_out) if ti + to > int(self.config['log_traffic']) * 1024: self.error_log("%iK in %iK out for <%s> (descend %s)" % ( ti / 1024, to / 1024, e_url(self.test_uri), str(self.descend))) self.response_start(b"200", b"OK", [ (b"Content-Type", formatter.content_type()), (b"Cache-Control", b"max-age=60, must-revalidate")]) if self.check_name: display_resource = top_resource.subreqs.get(self.check_name, top_resource) else: display_resource = top_resource formatter.bind_resource(display_resource) top_resource.check()
def slack_run(webui: "RedWebUi") -> None: """Handle a slack request.""" slack_response_uri = webui.body_args.get("response_url", [""])[0].strip() formatter = slack.SlackFormatter(webui.config, None, webui.output, slack_uri=slack_response_uri) webui.test_uri = webui.body_args.get("text", [""])[0].strip() webui.exchange.response_start( b"200", b"OK", [ (b"Content-Type", formatter.content_type()), (b"Cache-Control", b"max-age=300"), ], ) webui.output( json.dumps({ "response_type": "ephemeral", "text": f"_Checking_ {webui.test_uri} _..._", })) webui.exchange.response_done([]) top_resource = HttpResource(webui.config) top_resource.set_request(webui.test_uri, req_hdrs=webui.req_hdrs) formatter.bind_resource(top_resource) if not verify_slack_secret(webui): webui.error_response( formatter, b"403", b"Forbidden", "Incorrect Slack Authentication.", "Bad slack token.", ) return webui.timeout = thor.schedule(int(webui.config["max_runtime"]), formatter.timeout) @thor.events.on(formatter) def formatter_done() -> None: if webui.timeout: webui.timeout.delete() webui.timeout = None save_test(webui, top_resource) top_resource.check()
def continue_test( self, top_resource: HttpResource, formatter: Formatter, extra_headers: RawHeaderListType = [], ) -> None: "Preliminary checks are done; actually run the test." @thor.events.on(formatter) def formatter_done() -> None: if self.timeout: self.timeout.delete() self.timeout = None self.exchange.response_done([]) save_test(self, top_resource) # log excessive traffic ti = sum( [i.transfer_in for i, t in top_resource.linked], top_resource.transfer_in, ) to = sum( [i.transfer_out for i, t in top_resource.linked], top_resource.transfer_out, ) if ti + to > int(self.config["log_traffic"]) * 1024: self.error_log( f"{ti / 1024:n}K in {to / 1024:n}K out for <{e_url(self.test_uri)}> (descend {self.descend})" ) self.exchange.response_start( b"200", b"OK", [ (b"Content-Type", formatter.content_type()), (b"Cache-Control", b"max-age=60, must-revalidate"), ] + extra_headers, ) if self.check_name: display_resource = top_resource.subreqs.get( self.check_name, top_resource) else: display_resource = top_resource formatter.bind_resource(display_resource) top_resource.check()
def continue_test(self, top_resource: HttpResource, formatter: Formatter) -> None: "Preliminary checks are done; actually run the test." @thor.events.on(formatter) def formatter_done() -> None: self.response_done([]) if self.test_id: try: tmp_file = gzip.open(self.save_path, "w") pickle.dump(top_resource, tmp_file) tmp_file.close() except (IOError, zlib.error, pickle.PickleError): pass # we don't cry if we can't store it. # log excessive traffic ti = sum( [i.transfer_in for i, t in top_resource.linked], top_resource.transfer_in, ) to = sum( [i.transfer_out for i, t in top_resource.linked], top_resource.transfer_out, ) if ti + to > int(self.config["log_traffic"]) * 1024: self.error_log("%iK in %iK out for <%s> (descend %s)" % (ti / 1024, to / 1024, e_url( self.test_uri), str(self.descend))) self.response_start( b"200", b"OK", [ (b"Content-Type", formatter.content_type()), (b"Cache-Control", b"max-age=60, must-revalidate"), ], ) if self.check_name: display_resource = top_resource.subreqs.get( self.check_name, top_resource) else: display_resource = top_resource formatter.bind_resource(display_resource) top_resource.check()
def run_test(self) -> None: """Test a URI.""" if self.config.save_dir and os.path.exists(self.config.save_dir): try: fd, path = tempfile.mkstemp(prefix='', dir=self.config.save_dir) test_id = os.path.split(path)[1] except (OSError, IOError): # Don't try to store it. test_id = None else: test_id = None top_resource = HttpResource(descend=self.descend) self.timeout = thor.schedule(self.config.max_runtime, self.timeoutError, top_resource.show_task_map) top_resource.set_request(self.test_uri, req_hdrs=self.req_hdrs) formatter = find_formatter(self.format, 'html', self.descend)( self.ui_uri, self.config.lang, self.output, allow_save=test_id, is_saved=False, test_id=test_id, descend=self.descend) content_type = "%s; charset=%s" % (formatter.media_type, self.config.charset) if self.check_name: display_resource = top_resource.subreqs.get(self.check_name, top_resource) else: display_resource = top_resource referers = [] for hdr, value in self.req_hdrs: if hdr.lower() == 'referer': referers.append(value) referer_error = None if len(referers) > 1: referer_error = "Multiple referers not allowed." if referers and urlsplit(referers[0]).hostname in self.config.referer_spam_domains: referer_error = "Referer not allowed." if referer_error: self.response_start(b"403", b"Forbidden", [ (b"Content-Type", content_type.encode('ascii')), (b"Cache-Control", b"max-age=360, must-revalidate")]) formatter.start_output() formatter.error_output(referer_error) self.response_done([]) return if not self.robots_precheck(self.test_uri): self.response_start(b"502", b"Gateway Error", [ (b"Content-Type", content_type.encode('ascii')), (b"Cache-Control", b"max-age=60, must-revalidate")]) formatter.start_output() formatter.error_output("Forbidden by robots.txt.") self.response_done([]) return @thor.events.on(formatter) def formatter_done() -> None: self.response_done([]) if test_id: try: tmp_file = gzip.open(path, 'w') pickle.dump(top_resource, tmp_file) tmp_file.close() except (IOError, zlib.error, pickle.PickleError): pass # we don't cry if we can't store it. ti = sum([i.transfer_in for i, t in top_resource.linked], top_resource.transfer_in) to = sum([i.transfer_out for i, t in top_resource.linked], top_resource.transfer_out) if ti + to > self.config.log_traffic: self.error_log("%iK in %iK out for <%s> (descend %s)" % ( ti / 1024, to / 1024, e_url(self.test_uri), str(self.descend))) self.response_start(b"200", b"OK", [ (b"Content-Type", content_type.encode('ascii')), (b"Cache-Control", b"max-age=60, must-revalidate")]) formatter.bind_resource(display_resource) top_resource.check()
def run_test(self) -> None: """Test a URI.""" if self.config.save_dir and os.path.exists(self.config.save_dir): try: fd, path = tempfile.mkstemp(prefix='', dir=self.config.save_dir) test_id = os.path.split(path)[1] except (OSError, IOError): # Don't try to store it. test_id = None else: test_id = None top_resource = HttpResource(descend=self.descend) self.timeout = thor.schedule(self.config.max_runtime, self.timeoutError, top_resource.show_task_map) top_resource.set_request(self.test_uri, req_hdrs=self.req_hdrs) formatter = find_formatter(self.format, 'html', self.descend)(self.ui_uri, self.config.lang, self.output, allow_save=test_id, is_saved=False, test_id=test_id, descend=self.descend) content_type = "%s; charset=%s" % (formatter.media_type, self.config.charset) if self.check_name: display_resource = top_resource.subreqs.get( self.check_name, top_resource) else: display_resource = top_resource referers = [] for hdr, value in self.req_hdrs: if hdr.lower() == 'referer': referers.append(value) referer_error = None if len(referers) > 1: referer_error = "Multiple referers not allowed." if referers and urlsplit( referers[0]).hostname in self.config.referer_spam_domains: referer_error = "Referer not allowed." if referer_error: self.response_start( b"403", b"Forbidden", [(b"Content-Type", content_type.encode('ascii')), (b"Cache-Control", b"max-age=360, must-revalidate")]) formatter.start_output() formatter.error_output(referer_error) self.response_done([]) return if not self.robots_precheck(self.test_uri): self.response_start( b"502", b"Gateway Error", [(b"Content-Type", content_type.encode('ascii')), (b"Cache-Control", b"max-age=60, must-revalidate")]) formatter.start_output() formatter.error_output("Forbidden by robots.txt.") self.response_done([]) return @thor.events.on(formatter) def formatter_done() -> None: self.response_done([]) if test_id: try: tmp_file = gzip.open(path, 'w') pickle.dump(top_resource, tmp_file) tmp_file.close() except (IOError, zlib.error, pickle.PickleError): pass # we don't cry if we can't store it. ti = sum([i.transfer_in for i, t in top_resource.linked], top_resource.transfer_in) to = sum([i.transfer_out for i, t in top_resource.linked], top_resource.transfer_out) if ti + to > self.config.log_traffic: self.error_log("%iK in %iK out for <%s> (descend %s)" % (ti / 1024, to / 1024, e_url( self.test_uri), str(self.descend))) self.response_start( b"200", b"OK", [(b"Content-Type", content_type.encode('ascii')), (b"Cache-Control", b"max-age=60, must-revalidate")]) formatter.bind_resource(display_resource) top_resource.check()