def handle_response(self, msg):
# First see if we need to show the HTTPS user agreement/certificate download
client_ip = msg.flow.client_conn.address.address[0]
router_ip = global_config["router_IPs"]["erasure"]
if generate_trust(msg, client_ip, router_ip):
return
# Only worry about images
content_type = " ".join(msg.headers["content-type"])
if msg.code != 200 or content_type is None or not ("image/jpeg" in content_type or "image/webp" in content_type or "image/png" in content_type):
return
try:
req = msg.flow.request
url = u"{}://{}{}".format(req.get_scheme(), u"".join(req.headers["host"]), req.path)
ext = "JPEG" if "image/jpeg" in content_type else "PNG" if "image/png" in content_type else "WEBP"
content = msg.get_decoded_content()
if len(content) < 100:
return
censored = processedFace(content, ext)
msg.content = censored or msg.content
# Force uncompressed response
if censored:
log.info(u"Faces found in image: {}".format(url))
msg.headers["content-encoding"] = [""]
# Don't cache
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
except Exception as e:
log.exception(u"<{}> processing {} ".format(type(e).__name__, url))
def handle_response(self, msg):
# First see if we need to show the HTTPS user agreement/certificate download
client_ip = msg.flow.client_conn.address.address[0]
router_ip = global_config["router_IPs"]["similar"]
if generate_trust(msg, client_ip, router_ip):
return
is_image = (msg.headers["content-type"] == ["image/jpeg"] or msg.headers["content-type"] == ["image/png"] or msg.headers["content-type"] == ["image/webp"] or msg.headers["content-type"] == ["image/gif"]) and msg.code == 200 and not msg.flow.request.headers["X-Do-Not-Replace"] and len(msg.content) > options["smallest_image"]
global images_processed
global images_pending
if is_image: images_processed += 1
should_process = images_processed % options["frequency"] == 0
if is_image and should_process:
images_pending += 1
req = msg.flow.request
url = u"{}://{}{}".format(req.get_scheme(), u"".join(req.headers["host"]), req.path)
if images_pending > 10:
log.info("--- {} images in the queue!".format(images_pending))
try:
self.process_image(msg)
except Exception as e:
log.exception(u"<{}> processing {} ".format(type(e).__name__, url))
else:
msg.reply()
def handle_response(self, msg):
# Process replies from Internet servers to clients
# ------------------------------------------------
# First see if we need to show the HTTPS user agreement/certificate download
client_ip = msg.flow.client_conn.address.address[0]
router_ip = global_config["router_IPs"]["blackout"]
if generate_trust(msg, client_ip, router_ip):
return
content_type = " ".join(msg.headers["content-type"])
if msg.code != 301 and msg.code != 302 and content_type is not None and "text/html" in content_type:
try:
self.process_html(msg)
except Exception as e:
log.exception(e)
def handle_response(self, msg):
# Process replies from Internet servers to clients
# ------------------------------------------------
# First see if we need to show the HTTPS user agreement/certificate download
client_ip = msg.flow.client_conn.address.address[0]
router_ip = global_config["router_IPs"]["swap"]
if generate_trust(msg, client_ip, router_ip):
return
# Only worry about HTML for now
if msg.code != 200:
return
content_type = " ".join(msg.headers["content-type"])
content_headers = [x.strip() for x in content_type.split(";")]
charset = "utf-8"
for head in content_headers:
if head.startswith("charset="):
charset = head[8:].lower()
req = msg.flow.request
url = "{}://{}{}".format(req.get_scheme(), "".join(req.headers["host"]), req.path)
if content_type is not None and "text/html" in content_type:
# Decode contents (if gzip'ed)
contents = msg.get_decoded_content()
client_ip = msg.flow.client_conn.address.address[0]
hostname, mac = get_user_info(client_ip, global_config["router_IPs"]["swap"]) or client_ip
user = {"mac": mac, "ip": client_ip, "hostname": hostname}
t1 = time.time()
replacements = load_replacements(mac, url, hostname)
t2 = time.time()
#print "Loaded replacements in {}ms".format((t2-t1)*1000)
msg.content = process_as_html(user, url, contents, charset, replacements)
# Force uncompressed response
msg.headers["content-encoding"] = [""]
# Force unicode
msg.content = msg.content.encode("utf-8")
msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
if content_type is not None and ("image/jpeg" in content_type or "image/webp" in content_type or "image/png" in content_type):
filetype = "jpeg" if "jpeg" in content_type else "webp" if "webp" in content_type else "png"
client_ip = msg.flow.client_conn.address.address[0]
hostname, mac = get_user_info(client_ip, global_config["router_IPs"]["swap"]) or client_ip
user = {"mac": mac, "ip": client_ip, "hostname": hostname}
contents = msg.get_decoded_content()
if len(contents) < 10:
return
msg.content = process_image(user, url, contents, filetype)
msg.headers["content-encoding"] = [""]
# Handle JSON
if "https://twitter.com" in url and content_type is not None and ("json" in content_type or "javascript" in content_type):
try:
j = json.loads(msg.get_decoded_content(), encoding = charset)
except ValueError:
# Not really JSON
#print "this is not json1!!"
return
client_ip = msg.flow.client_conn.address.address[0]
hostname, mac = get_user_info(client_ip, global_config["router_IPs"]["swap"]) or client_ip
user = {"mac": mac, "ip": client_ip, "hostname": hostname}
replacements = load_replacements(mac, url, hostname)
process_html_in_json(user, url, j, charset, replacements)
msg.content = json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '), encoding="utf-8")
# Force uncompressed response
msg.headers["content-encoding"] = [""]
# Force unicode
msg.content = msg.content.encode("utf-8")
#msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
msg.headers["content-type"] = ["text/html; charset=utf-8"]
# Never cache response
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
def handle_response(self, msg):
# Process replies from Internet servers to clients
# ------------------------------------------------
# First see if we need to show the HTTPS user agreement/certificate download
client_ip = msg.flow.client_conn.address.address[0]
router_ip = global_config["router_IPs"]["local"]
if generate_trust(msg, client_ip, router_ip):
return
content_type = " ".join(msg.headers["content-type"])
if content_type is None or "text/html" not in content_type:
return
if avoid_captive_portal(msg):
return
try:
content_type = " ".join(msg.headers["content-type"])
msg.flow.request.host = "".join(msg.flow.request.headers["host"])
if msg.code != 301 and msg.code != 302 and content_type is not None and "text/html" in content_type and msg.flow.request.host not in ["192.168.1.128", "127.0.0.1", "localhost"]:
# Try to do a local DNS lookup and search by IP for more accurate results
try:
#query = socket.gethostbyaddr(msg.request.host)[2][0]
query = socket.getaddrinfo(msg.flow.request.host, 80)[0][4][0] # I think this is more reliable
except:
query = msg.flow.request.host
r = requests.get("http://freegeoip.net/json/" + query)
log.debug(u"{} - {}".format(msg.flow.request.host, r.content))
try:
j = json.loads(r.content)
country_code = j["country_code"].lower()
region_code = j["region_code"].lower()
# Append "the" before certain countries to sound more natural
if j["country_name"] in ["United States", "United Kingdom", "Netherlands"] and j["region_name"] == "" and j["city"] == "":
j["country_name"] = "the {}".format(j["country_name"])
if country_code not in local_country_codes:
# Just show a flag
notes = ""
if j["region_name"] != "":
if j["city"] != "":
notes = u"It is hosted in {}, {}, {}.".format(j["city"], j["region_name"], j["country_name"])
else:
notes = u"It is hosted in {}, {}.".format(j["region_name"], j["country_name"])
else:
notes = u"It is hosted in {}.".format(j["country_name"])
#notes = u"<span style='font-size: 50%'>" + notes + u"</span>"
flag = country_code
if country_code == "us" and region_code != "":
flag = u"{}-{}".format(country_code, region_code)
template = template_env.get_template('local/notlocal.html')
msg.content = template.render(flag=flag, host=msg.flow.request.host, notes=notes)
#msg.content = u"<html><body style='background: url(http://{}:{}/flags/{}.png); background-size: 100%;'><div style='width: 900px; height: 200px; margin: auto; position: absolute; left:0; right:0; top:0; bottom:0; text-align: center; font-size: 36pt; font-family: sans-serif; font-weight: bold; color: white; line-height: 1.5em; text-shadow: black 0 0 40px;'><div style='background: rgba(0,0,0,.5); width: auto;'>{}<br>IS NOT LOCAL<br>{}</div></div></body></html>".format(options["static-server-host"], options["static-server-port"], flag, msg.flow.request.host.upper(), extras)
# Force unicode
msg.content = msg.content.encode("utf-8")
msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
#if len(msg.headers["content-encoding"]) > 0:
# msg.encode(msg.headers["content-encoding"][0])
# Force uncompressed response
msg.headers["content-encoding"] = [""]
# Don't cache
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
log.info(u"NOT local: <{}>".format(msg.flow.request.host))
else:
log.info(u"LOCAL: <{}>".format(msg.flow.request.host))
except ValueError as e:
#template = template_env.get_template('local/error.html')
template = template_env.get_template('local/notlocal.html')
msg.content = template.render(host=msg.flow.request.host, flag="missing")
#msg.content = "<html><body style='background: url(http://{}:{}/flags/missing.png); background-size: 100%;'><div style='width: 900px; height: 200px; margin: auto; position: absolute; left:0; right:0; top:0; bottom:0; text-align: center; font-size: 36pt; font-family: sans-serif; font-weight: bold; color: white; line-height: 1.5em; text-shadow: black 0 0 40px;'><div style='background: rgba(0,0,0,.5); width: auto;'>I DON'T KNOW WHERE I AM<br><span style='font-size: 50%; line-height: 1.75em;'>Check back later to find out if<br>{}<br>is local.</span></div></div></body></html>".format(options["static-server-host"], options["static-server-port"], msg.flow.request.host.upper())
msg.headers["content-encoding"] = [""]
# Don't cache
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
log.exception(u"<{}> parsing JSON for {} ".format(type(e).__name__, msg.flow.request.host))
except Exception as e:
log.exception(u"<{}> checking {} ".format(type(e).__name__, msg.flow.request.host))
def handle_response(self, msg):
# First see if we need to show the HTTPS user agreement/certificate download
client_ip = msg.flow.client_conn.address.address[0]
router_ip = global_config["router_IPs"]["surf"]
if generate_trust(msg, client_ip, router_ip):
return
if avoid_captive_portal(msg):
return
try:
# Only worry about HTML for now
content_type = " ".join(msg.headers["content-type"])
content_headers = [x.strip() for x in content_type.split(";")]
charset = None
for head in content_headers:
if head.startswith("charset="):
charset = head[8:].lower()
if content_type is not None and "text/html" in content_type:
req = msg.flow.request
client_ip = msg.flow.client_conn.address.host
url = u"{}://{}{}".format(req.get_scheme(), u"".join(req.headers["host"]), req.path)
#print msg.headers["content-encoding"]
#print dir(msg)
#print msg.get_decoded_content()
# Crawl a little bit if there aren't many pages in the queue yet
if len(waveQueue) < 10:
links = get_links(url, msg.get_decoded_content(), charset = charset)
if len(links) > 0:
l = choice(links)
for i in range(5):
try:
log.debug(u"Crawling {}".format(l))
r = requests.get(l)
waveQueue[l] = r.content.replace("</html>", "")
l = choice(get_links(l, r.content, None))
except requests.ConnectionError:
break
if req.headers["x-surf-changed"] or self.weather_params["FLAT"] or random() < self.weather_params["WAIT"]:
gifs = [ f for f in listdir(gif_dir) if isfile(join(gif_dir,f))]
g = choice(gifs)
# Tranquilo...
message = ""
if self.weather_params["FLAT"]:
message = "Not much happening here..."
else:
if self.weather_params["SIZE"] > 2:
message = "Big waves "
end = "!"
elif self.weather_params["SIZE"] > 1:
message = "A nice swell "
end = "."
else:
message = "A gentle swell "
end = "."
if self.weather_params["SLOP"] >= .1:
message += "but it's all blown out!"
elif self.weather_params["SLOP"] > .5:
message += "with a bit of chop" + end
else:
message += end
template = template_env.get_template("surf/tranquilo.html")
msg.content = (template.render(spots=surf_spots, current_spot = self.current_spot, background=g,
wind = self.weather_params["wind"], waves = self.weather_params["waves"], message = message,
flat = self.weather_params["FLAT"], redirect = url)).encode("utf-8")
#msg.content = "<html><head><META HTTP-EQUIV='CACHE-CONTROL' CONTENT='NO-CACHE'></head><body style='margin: 0px;'><a href='javascript:window.location.reload()'><img src='http://media0.giphy.com/media/u538oVJPQ0Rzi/200.gif' style='border: 0; width: 100%; height: 100%'></a></body></html>"
msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
msg.headers["content-encoding"] = []
# Allow any script
del(msg.headers["content-security-policy"])
else:
new = not waveQueue.has_key(url)
contents = msg.get_decoded_content().replace("</html>", "")
waveQueue[url] = contents
bigWave = {}
output = ""
links = get_links(url, contents, charset = charset)
if self.weather_params["SIZE"] > 1 and len(links) > 0:
# Get links, download one or two at random, append
for i in range(0, int((self.weather_params["SIZE"] - 1) * 3)):
if random() < 0.8:
l = choice(links)
log.debug(u"Adding {}".format(l))
try:
r = requests.get(l)
waveQueue[l] = r.content.replace("</html>", "")
bigWave[l] = waveQueue[l]
except requests.ConnectionError:
log.info(u"Couldn't get link {}".format(l))
survivors = {}
keys = waveQueue.keys()
shuffle(keys)
i = 0
for u in keys:
i += 2
# Always output the current page, maybe some extra junk
if url == u or (random() < self.weather_params["SLOP"] and i < 50):
thispage = waveQueue[u]
insertion_point = randint(0, len(output))
cut_start = randint(0, len(thispage))
cut_stop = randint(cut_start, len(thispage))
if url == u:
cut_start = 0
cut_stop = len(thispage)
#if self.weather_params["SIZE"] < 1:
# cut_stop = len(thispage) * uniform(self.weather_params["SIZE"],1)
log.debug(u"Inserting [{}:{}] from {} at {}".format(cut_start, cut_stop, u, insertion_point))
output = output[:insertion_point] + thispage[cut_start:cut_stop] + output[insertion_point:]
if url == u and len(bigWave) > 0:
for l in bigWave:
output += bigWave[l]
# Give this page a chance to surface again
if new or random() < self.weather_params["REVERBERATE"]:
survivors[u] = waveQueue[u]
else:
survivors[u] = waveQueue[u]
msg.content = output
msg.headers["content-encoding"] = []
# Don't cache
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
# Update the list
waveQueue.clear()
for u in survivors:
waveQueue[u] = survivors[u]
# Don't cache
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
except Exception as e:
log.exception(e)
#print e
#print traceback.format_exc()
msg.reply()
def handle_response(self, msg):
# Process replies from Internet servers to clients
# ------------------------------------------------
# First see if we need to show the HTTPS user agreement/certificate download
client_ip = msg.flow.client_conn.address.address[0]
router_ip = global_config["router_IPs"]["free"]
if generate_trust(msg, client_ip, router_ip):
return
# Only worry about HTML for now
content_type = " ".join(msg.headers["content-type"])
content_headers = [x.strip() for x in content_type.split(";")]
charset = None
req = msg.flow.request
url = u"{}://{}{}".format(req.get_scheme(), u"".join(req.headers["host"]), req.path)
for head in content_headers:
if head.startswith("charset="):
charset = head[8:].lower()
if content_type is not None and "jpg" in content_type or "png" in content_type or "jpeg" in content_type:
# Check to see if this is an ad
if adblock_filter.match(url):
with open("../static/img/face-dot.png") as f:
msg.content = f.read()
# Force uncompressed response
msg.headers["content-encoding"] = [""]
msg.headers["content-type"] = ["image/png"]
log.info(u"Replacing ad image: {}".format(url))
# Never cache modified response
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
elif content_type is not None and "text/html" in content_type:
if adblock_filter.match(url):
msg.content = "!!!"
return
# Decode contents (if gzip'ed)
contents = msg.get_decoded_content()
#contents = "<p>€32</p>"
#charset = "utf-8"
soup = process_as_html(contents, charset = charset)
msg.content = soup
# Force uncompressed response
msg.headers["content-encoding"] = [""]
# Force unicode
msg.content = msg.content.encode("utf-8")
msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
# Never cache modified response
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
# Handle JSON
elif content_type is not None and "json" in content_type or "javascript" in content_type:
if adblock_filter.match(url):
msg.content = "!!!"
return
return
try:
j = json.loads(msg.get_decoded_content(), encoding = charset)
except ValueError:
# Not really JSON
#print "this is not jsgon1!!"
return
return
process_html_in_json(j, charset)
msg.content = json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '), encoding="utf-8")
# Force uncompressed response
msg.headers["content-encoding"] = [""]
# Force unicode
msg.content = msg.content.encode("utf-8")
msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
# Never cache modified response
msg.headers["Pragma"] = ["no-cache"]
msg.headers["Cache-Control"] = ["no-cache, no-store"]
