def run_test(self):
"""Test a URI."""
if save_dir and os.path.exists(save_dir):
try:
fd, path = tempfile.mkstemp(prefix='', dir=save_dir)
test_id = os.path.split(path)[1]
except (OSError, IOError):
# Don't try to store it.
test_id = None
else:
test_id = None
formatter = find_formatter(self.format, 'html', self.descend)(
self.base_uri, self.test_uri, self.req_hdrs, lang,
self.output, allow_save=test_id, is_saved=False,
test_id=test_id, descend=self.descend
)
self.response_start(
"200", "OK", [
("Content-Type", "%s; charset=%s" % (
formatter.media_type, charset)),
("Cache-Control", "max-age=60, must-revalidate")
])
ired = HttpResource(
self.test_uri,
req_hdrs=self.req_hdrs,
status_cb=formatter.status,
body_procs=[formatter.feed],
descend=self.descend
)
# sys.stdout.write(pickle.dumps(ired))
formatter.start_output()
def done():
if self.check_type:
# TODO: catch errors
state = ired.subreqs.get(self.check_type, None)
else:
state = ired
formatter.set_state(state)
formatter.finish_output()
self.response_done([])
if test_id:
try:
tmp_file = gzip.open(path, 'w')
pickle.dump(ired, tmp_file)
tmp_file.close()
except (IOError, zlib.error, pickle.PickleError):
pass # we don't cry if we can't store it.
# objgraph.show_growth()
ired.run(done)
def run_test(self):
"""Test a URI."""
if save_dir and os.path.exists(save_dir):
try:
fd, path = tempfile.mkstemp(prefix='', dir=save_dir)
test_id = os.path.split(path)[1]
except (OSError, IOError):
# Don't try to store it.
test_id = None
else:
test_id = None
formatter = find_formatter(self.format, 'html', self.descend)(
self.base_uri, self.test_uri, self.req_hdrs, lang,
self.output, allow_save=test_id, is_saved=False,
check_type=self.check_type, test_id=test_id, descend=self.descend
)
referers = []
for hdr, value in self.req_hdrs:
if hdr.lower() == 'referer':
referers.append(value)
referer_error = None
if len(referers) > 1:
referer_error = "Multiple referers not allowed."
if referers and urlsplit(referers[0]).hostname in referer_spam_domains:
referer_error = "Referer now allowed."
if referer_error:
self.response_start(
"403", "Forbidden", [
("Content-Type", "%s; charset=%s" % (
formatter.media_type, charset)),
("Cache-Control", "max-age=360, must-revalidate")
])
formatter.start_output()
self.output(error_template % referer_error)
self.response_done([])
return
if not self.robots_precheck(self.test_uri):
self.response_start(
"502", "Gateway Error", [
("Content-Type", "%s; charset=%s" % (
formatter.media_type, charset)),
("Cache-Control", "max-age=60, must-revalidate")
])
formatter.start_output()
self.output(error_template % "Forbidden by robots.txt.")
self.response_done([])
return
self.response_start(
"200", "OK", [
("Content-Type", "%s; charset=%s" % (
formatter.media_type, charset)),
("Cache-Control", "max-age=60, must-revalidate")
])
resource = HttpResource(
self.test_uri,
req_hdrs=self.req_hdrs,
status_cb=formatter.status,
body_procs=[formatter.feed],
descend=self.descend
)
# sys.stdout.write(pickle.dumps(resource))
formatter.start_output()
def done():
if self.check_type:
state = resource.subreqs.get(self.check_type, resource)
else:
state = resource
formatter.set_state(state)
formatter.finish_output()
self.response_done([])
if test_id:
try:
tmp_file = gzip.open(path, 'w')
pickle.dump(resource, tmp_file)
tmp_file.close()
except (IOError, zlib.error, pickle.PickleError):
pass # we don't cry if we can't store it.
# objgraph.show_growth()
ti = sum([i.transfer_in for i,t in resource.linked], resource.transfer_in)
to = sum([i.transfer_out for i,t in resource.linked], resource.transfer_out)
if ti + to > log_traffic:
sys.stderr.write("%iK in %iK out for <%s> (descend %s)" % (
ti / 1024,
to / 1024,
self.test_uri,
str(self.descend)
))
resource.run(done)
def run_test(self):
"""Test a URI."""
if save_dir and os.path.exists(save_dir):
try:
fd, path = tempfile.mkstemp(prefix="", dir=save_dir)
test_id = os.path.split(path)[1]
except (OSError, IOError):
# Don't try to store it.
test_id = None
else:
test_id = None
formatter = find_formatter(self.format, "html", self.descend)(
self.base_uri,
self.test_uri,
self.req_hdrs,
lang,
self.output,
allow_save=test_id,
is_saved=False,
test_id=test_id,
descend=self.descend,
)
referers = []
for hdr, value in self.req_hdrs:
if hdr.lower() == "referer":
referers.append(value)
referer_error = None
if len(referers) > 1:
referer_error = "Multiple referers not allowed."
if referers and urlsplit(referers[0]).hostname in referer_spam_domains:
referer_error = "Referer now allowed."
if referer_error:
self.response_start(
"403",
"Forbidden",
[
("Content-Type", "%s; charset=%s" % (formatter.media_type, charset)),
("Cache-Control", "max-age=360, must-revalidate"),
],
)
formatter.start_output()
self.output(error_template % referer_error)
self.response_done([])
return
if not self.robots_precheck(self.test_uri):
self.response_start(
"502",
"Gateway Error",
[
("Content-Type", "%s; charset=%s" % (formatter.media_type, charset)),
("Cache-Control", "max-age=60, must-revalidate"),
],
)
formatter.start_output()
self.output(error_template % "Forbidden by robots.txt.")
self.response_done([])
return
self.response_start(
"200",
"OK",
[
("Content-Type", "%s; charset=%s" % (formatter.media_type, charset)),
("Cache-Control", "max-age=60, must-revalidate"),
],
)
ired = HttpResource(
self.test_uri,
req_hdrs=self.req_hdrs,
status_cb=formatter.status,
body_procs=[formatter.feed],
descend=self.descend,
)
# sys.stdout.write(pickle.dumps(ired))
formatter.start_output()
def done():
if self.check_type:
# TODO: catch errors
state = ired.subreqs.get(self.check_type, None)
else:
state = ired
formatter.set_state(state)
formatter.finish_output()
self.response_done([])
if test_id:
try:
tmp_file = gzip.open(path, "w")
pickle.dump(ired, tmp_file)
tmp_file.close()
except (IOError, zlib.error, pickle.PickleError):
pass # we don't cry if we can't store it.
# objgraph.show_growth()
ired.run(done)
