示例#1
0
    def handle_response(self, msg):
        # Process replies from Internet servers to clients
        # ------------------------------------------------
        # First see if we need to show the HTTPS user agreement/certificate download
        client_ip = msg.flow.client_conn.address.address[0]
        router_ip = global_config["router_IPs"]["local"]
        if generate_trust(msg, client_ip, router_ip):
            return

        content_type = " ".join(msg.headers["content-type"])
        if content_type is None or "text/html" not in content_type:
            return

        if avoid_captive_portal(msg):
            return

        try:
            
            content_type = " ".join(msg.headers["content-type"])
            
            msg.flow.request.host = "".join(msg.flow.request.headers["host"])

            if msg.code != 301 and msg.code != 302 and content_type is not None and "text/html" in content_type and msg.flow.request.host not in ["192.168.1.128", "127.0.0.1", "localhost"]:
                # Try to do a local DNS lookup and search by IP for more accurate results
                try:
                    #query = socket.gethostbyaddr(msg.request.host)[2][0]
                    query = socket.getaddrinfo(msg.flow.request.host, 80)[0][4][0]  # I think this is more reliable
                except:
                    query = msg.flow.request.host

            
                r = requests.get("http://freegeoip.net/json/" + query)
                log.debug(u"{} - {}".format(msg.flow.request.host, r.content))

                try:
                    j = json.loads(r.content)
                    country_code = j["country_code"].lower()
                    region_code = j["region_code"].lower()
                
                    # Append "the" before certain countries to sound more natural
                    if j["country_name"] in ["United States", "United Kingdom", "Netherlands"] and j["region_name"] == "" and j["city"] == "":
                        j["country_name"] = "the {}".format(j["country_name"])

                    if country_code not in local_country_codes:
                        # Just show a flag
                        notes = ""
                        if j["region_name"] != "":
                            if j["city"] != "":
                                notes = u"It is hosted in {}, {}, {}.".format(j["city"], j["region_name"], j["country_name"])
                            else:
                                notes = u"It is hosted in {}, {}.".format(j["region_name"], j["country_name"])
                        else:
                            notes = u"It is hosted in {}.".format(j["country_name"])

                        #notes = u"<span style='font-size: 50%'>" + notes + u"</span>"

                        flag = country_code
                        if country_code == "us" and region_code != "":
                            flag = u"{}-{}".format(country_code, region_code)

                        template = template_env.get_template('local/notlocal.html')
                        msg.content = template.render(flag=flag, host=msg.flow.request.host, notes=notes)

                        #msg.content = u"<html><body style='background: url(http://{}:{}/flags/{}.png); background-size: 100%;'><div style='width: 900px; height: 200px; margin: auto; position: absolute; left:0; right:0; top:0; bottom:0; text-align: center; font-size: 36pt; font-family: sans-serif; font-weight: bold; color: white; line-height: 1.5em; text-shadow: black 0 0 40px;'><div style='background: rgba(0,0,0,.5); width: auto;'>{}<br>IS NOT LOCAL<br>{}</div></div></body></html>".format(options["static-server-host"], options["static-server-port"], flag, msg.flow.request.host.upper(), extras)
                    
                        # Force unicode
                        msg.content = msg.content.encode("utf-8")
                        msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
                    
                        #if len(msg.headers["content-encoding"]) > 0:
                        #   msg.encode(msg.headers["content-encoding"][0])
                        # Force uncompressed response
                        msg.headers["content-encoding"] = [""]
                        
                        # Don't cache
                        msg.headers["Pragma"] = ["no-cache"]
                        msg.headers["Cache-Control"] = ["no-cache, no-store"]

                        log.info(u"NOT local: <{}>".format(msg.flow.request.host))
                    else:
                        log.info(u"LOCAL: <{}>".format(msg.flow.request.host))
                except ValueError as e:
                    #template = template_env.get_template('local/error.html')
                    template = template_env.get_template('local/notlocal.html')
                    msg.content = template.render(host=msg.flow.request.host, flag="missing")

                    #msg.content = "<html><body style='background: url(http://{}:{}/flags/missing.png); background-size: 100%;'><div style='width: 900px; height: 200px; margin: auto; position: absolute; left:0; right:0; top:0; bottom:0; text-align: center; font-size: 36pt; font-family: sans-serif; font-weight: bold; color: white; line-height: 1.5em; text-shadow: black 0 0 40px;'><div style='background: rgba(0,0,0,.5); width: auto;'>I DON'T KNOW WHERE I AM<br><span style='font-size: 50%; line-height: 1.75em;'>Check back later to find out if<br>{}<br>is local.</span></div></div></body></html>".format(options["static-server-host"], options["static-server-port"], msg.flow.request.host.upper())
                    msg.headers["content-encoding"] = [""]
                    # Don't cache
                    msg.headers["Pragma"] = ["no-cache"]
                    msg.headers["Cache-Control"] = ["no-cache, no-store"]
                    
                    
                    log.exception(u"<{}> parsing JSON for {} ".format(type(e).__name__, msg.flow.request.host))
                    

        except Exception as e:
            log.exception(u"<{}> checking {} ".format(type(e).__name__, msg.flow.request.host))
    def process_html(self, msg):
        if avoid_captive_portal(msg):
            return


        # Check if the requested URL is already in the database
        req = msg.flow.request
        url = "{}://{}{}".format(req.get_scheme(), "".join(req.headers["host"]), req.path)
        
        db = sqlite3.connect("db/blackout.db")
        db.row_factory = sqlite3.Row
        cursor = db.cursor()
        cursor.execute('''SELECT * FROM resources WHERE url=?''', (url,))
        resource = cursor.fetchone()

        client_ip = msg.flow.client_conn.address.address[0]

        if resource is None:
            # If not, add the URL to the database with current timestamp
            # and pass the page on as usual

            hostname = get_hostname(client_ip, global_config["router_IPs"]["blackout"]) or client_ip
            cursor.execute('''INSERT INTO resources(url, last_accessed, life_remaining, accessed_by) VALUES(?, ?, 0, ?)''',
                                                    (url, int(time()), hostname))
            db.commit()

        else:
            # If yes in the database, was it accessed within the past 24 hours?
            now = int(time())
            then = resource["last_accessed"]
            accessed_by = resource["accessed_by"]

            blackout_time = 86400 #86400 # 24 hours in seconds
            if now - then > blackout_time:
                # If not accessed in 24 hours, update the database with the current timestamp
                # and pass the page on as usual
                hostname = get_hostname(client_ip, global_config["router_IPs"]["blackout"]) or client_ip
                cursor.execute('''UPDATE resources SET last_accessed = ?, accessed_by = ? WHERE url = ?''', (now, hostname, url))
                db.commit()
            else:
                # If it was accessed in past 24 hours, display the blackout page with info
                then_date = datetime.fromtimestamp(then)
                now_date = datetime.fromtimestamp(now)
                available_date = then_date + timedelta(0, blackout_time)
                time_diff = now_date - then_date
                available_diff = available_date - now_date

                # Figure natural description for last access day
                accessed_day = ""
                if time_diff.days == 0:
                    accessed_day = "today"
                elif time_diff.days == 1:
                    accessed_day = "yesterday"
                else:
                    accessed_day = "{} days ago".format(time_diff.days)

                # Figure natural description for day page will be accessible again
                available_day = ""
                if available_diff.days == 0:
                    available_day = "today"
                elif available_diff.days == 1:
                    available_day = "tomorrow"
                else:
                    available_day = "in {} days".format(available_diff.days)

                blackout_diff = available_date - now_date
                minutes,seconds = divmod(blackout_diff.total_seconds(), 60)
                hours, minutes = divmod(minutes, 60)

                then_string = "{} at {}".format(accessed_day, then_date.strftime("%H:%M"))
                
                template = template_env.get_template("blackout/notavailable.html")

                msg.content = template.render(url=url, access_time = then_string, hours=int(hours), minutes=int(minutes), seconds=int(seconds), accessed_by=accessed_by)

                
                # Force unicode
                msg.content = msg.content.encode("utf-8")
                msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
            
                # Force uncompressed response
                msg.headers["content-encoding"] = [""]
                
                # Don't cache
                msg.headers["Pragma"] = ["no-cache"]
                msg.headers["Cache-Control"] = ["no-cache, no-store"]

                # Allow any script
                del(msg.headers["content-security-policy"]) 
        db.close()
示例#3
0
    def handle_response(self, msg):
        # First see if we need to show the HTTPS user agreement/certificate download
        client_ip = msg.flow.client_conn.address.address[0]
        router_ip = global_config["router_IPs"]["surf"]
        if generate_trust(msg, client_ip, router_ip):
            return

        if avoid_captive_portal(msg):
            return
        
        try:
            # Only worry about HTML for now
            content_type = " ".join(msg.headers["content-type"])
            content_headers = [x.strip() for x in content_type.split(";")]
            charset = None
            for head in content_headers:
                if head.startswith("charset="):
                    charset = head[8:].lower()

            if content_type is not None and "text/html" in content_type:
                req = msg.flow.request
                client_ip = msg.flow.client_conn.address.host
                url = u"{}://{}{}".format(req.get_scheme(), u"".join(req.headers["host"]), req.path)

                #print msg.headers["content-encoding"]
                #print dir(msg)
                #print msg.get_decoded_content()

                # Crawl a little bit if there aren't many pages in the queue yet
                if len(waveQueue) < 10:
                    links = get_links(url, msg.get_decoded_content(), charset = charset)
                    if len(links) > 0:
                        l = choice(links)
                        for i in range(5):
                            try:
                                log.debug(u"Crawling {}".format(l))
                                r = requests.get(l)
                                waveQueue[l] = r.content.replace("</html>", "")
                                l = choice(get_links(l, r.content, None))
                            except requests.ConnectionError:
                                break


                if req.headers["x-surf-changed"] or self.weather_params["FLAT"] or random() < self.weather_params["WAIT"]:
                    gifs = [ f for f in listdir(gif_dir) if isfile(join(gif_dir,f))]
                    g = choice(gifs)

                    # Tranquilo...
                    message = ""
                    if self.weather_params["FLAT"]:
                        message = "Not much happening here..."
                    else:
                        if self.weather_params["SIZE"] > 2:
                            message = "Big waves "
                            end = "!"
                        elif self.weather_params["SIZE"] > 1:
                            message = "A nice swell "
                            end = "."
                        else:
                            message = "A gentle swell "
                            end = "."
                        if self.weather_params["SLOP"] >= .1:
                            message += "but it's all blown out!"
                        elif self.weather_params["SLOP"] > .5:
                            message += "with a bit of chop" + end
                        else:
                            message += end

                    template = template_env.get_template("surf/tranquilo.html")
                    msg.content = (template.render(spots=surf_spots, current_spot = self.current_spot, background=g,
                                   wind = self.weather_params["wind"], waves = self.weather_params["waves"], message = message,
                                   flat = self.weather_params["FLAT"], redirect = url)).encode("utf-8")

                
                    #msg.content = "<html><head><META HTTP-EQUIV='CACHE-CONTROL' CONTENT='NO-CACHE'></head><body style='margin: 0px;'><a href='javascript:window.location.reload()'><img src='http://media0.giphy.com/media/u538oVJPQ0Rzi/200.gif' style='border: 0; width: 100%; height: 100%'></a></body></html>"
                    msg.headers["content-type"] = ["{}; charset=utf-8".format(msg.headers["content-type"][0])]
                    msg.headers["content-encoding"] =  []

                    # Allow any script
                    del(msg.headers["content-security-policy"]) 

                else:
                    new = not waveQueue.has_key(url)
                    contents = msg.get_decoded_content().replace("</html>", "")
                    waveQueue[url] = contents
                    bigWave = {}
                    output = ""

                    links = get_links(url, contents, charset = charset)
                    if self.weather_params["SIZE"] > 1 and len(links) > 0:
                        # Get links, download one or two at random, append
                        for i in range(0, int((self.weather_params["SIZE"] - 1) * 3)):
                            if random() < 0.8:
                                l = choice(links)
                                log.debug(u"Adding {}".format(l))
                                try:
                                    r = requests.get(l)
                                    waveQueue[l] = r.content.replace("</html>", "")
                                    bigWave[l] = waveQueue[l]
                                except requests.ConnectionError:
                                    log.info(u"Couldn't get link {}".format(l))

                    survivors = {}
                    keys = waveQueue.keys()
                    shuffle(keys)
                
                    i = 0
                    for u in keys:
                        i += 2
                        # Always output the current page, maybe some extra junk
                        if url == u or (random() < self.weather_params["SLOP"] and i < 50):
                            thispage = waveQueue[u]
                            insertion_point = randint(0, len(output))

                            cut_start = randint(0, len(thispage))
                            cut_stop  = randint(cut_start, len(thispage))
                            if url == u:
                                cut_start = 0
                                cut_stop = len(thispage)
                                #if self.weather_params["SIZE"] < 1:
                                #    cut_stop = len(thispage) * uniform(self.weather_params["SIZE"],1)
                            

                            log.debug(u"Inserting [{}:{}] from {} at {}".format(cut_start, cut_stop, u, insertion_point))
                            output = output[:insertion_point] + thispage[cut_start:cut_stop] + output[insertion_point:]

                            if url == u and len(bigWave) > 0:
                                for l in bigWave:
                                    output += bigWave[l]

                            # Give this page a chance to surface again
                            if new or random() < self.weather_params["REVERBERATE"]:
                                survivors[u] = waveQueue[u] 
                        else:
                            survivors[u] = waveQueue[u]

                    msg.content = output
                    msg.headers["content-encoding"] =  []
                    
                    # Don't cache
                    msg.headers["Pragma"] = ["no-cache"]
                    msg.headers["Cache-Control"] = ["no-cache, no-store"]

                    # Update the list
                    waveQueue.clear()
                    for u in survivors:
                        waveQueue[u] = survivors[u]

                # Don't cache
                msg.headers["Pragma"] = ["no-cache"]
                msg.headers["Cache-Control"] = ["no-cache, no-store"]

        except Exception as e:
            log.exception(e)
            #print e
            #print traceback.format_exc()

        msg.reply()