def _get_or_create_host(cls, hostname, create=True): try: return Host.get(Host.hostname == hostname), False except DoesNotExist: if create: return Host.create(hostname=hostname), True return None, None
def _start_upstream(self): self._buffer.seek(0) firstline = self._buffer.readline() match = re.match("(?:CONNECT) ([^:]+)(?:[:](\d+))? \w+", firstline) if match is None: return host = match.group(1) port = int(match.group(2) or 443) cachebrowsed = False try: Host.get(Host.hostname == host) cachebrowsed = True except Host.DoesNotExist: pass if cachebrowsed: #logger.info("[HTTPS] %s:%s <REJECTED>" % (host, port)) #self.connection.close_local() logger.info("[HTTPS] %s:%s <CACHEBROWSED>" % (host, port)) return self._connect_upstream(host, port) else: logger.info("[HTTPS] %s:%s <PROXYING>" % (host, port)) return self._connect_upstream(host, port)
def get_hosts(context, request): page = request.params.get('page', 0) num_per_page = request.params.get('num_per_page', 5) if num_per_page <= 0: result = Host.select() else: result = Host.select().paginate(page, num_per_page) hosts = [serialize_host(host) for host in result] request.reply(hosts)
def bootstrap(context, save, hostname): try: host_data = context.bootstrapper.lookup_host(hostname) except BootstrapError: logger.warning("No bootstrap information found for host '{}'".format(hostname)) return logger.info(json.dumps(host_data, indent=4)) if save: host = Host(**host_data) host.save()
def serverconnect(self, server_conn): hostname = server_conn.address.host # TODO (NOT FOR ALL SITE, BUT NEEDED FOR YOUTUBE) server_conn.sni = "" server_conn.cachebrowsed = False server_conn.cdn = None server_conn.cb_status_message = "" try: host = Host.get_or_bootstrap(hostname=hostname) except DoesNotExist: err = "Bootstrapping host {} failed".format(hostname) server_conn.cb_status_message = err return server_conn # Skip if host is not active if not host.is_active: server_conn.status = "Host not active" return server_conn cdn = host.cdn if not cdn.valid or not cdn.edge_server: err = "Host {} does not have a valid CDN".format(hostname) server_conn.cb_status_message = err return server_conn server_conn.address = Address((cdn.edge_server, server_conn.address.port)) server_conn.sni = "" server_conn.cachebrowsed = True server_conn.cdn = {"id": cdn.id, "name": cdn.name} return server_conn
def domain_list(self): """ List the active hosts """ hosts = Host.select() for host in hosts: self.send_line("%*s: %s" % (20, host.url, host.cdn.id))
def domain_list(self): """ List the active hosts """ hosts = Host.select() for host in hosts: self.send_line("%*s: %s" % (20, host.hostname, host.cdn.id))
def get_hosts(request): page = request.params.get('page', 0) num_per_page = request.params.get('num_per_page', 5) result = Host.select().paginate(page, num_per_page) # .order_by(Host.hostname) hosts = [serialize_host(host) for host in result] request.reply(hosts)
def add_host(context, request): host = Host() host.hostname = request.params.get('hostname', '') host.cdn = request.params.get('cdn', '') host.ssl = request.params.get('ssl', True) host.save(force_insert=True) request.reply()
def action_check_host(request): hostname = extract_url_hostname(request['host']) try: is_active = Host.get(Host.hostname == hostname).is_active except DoesNotExist: is_active = False return { 'result': 'active' if is_active else 'inactive', 'host': request['host'] }
def resolve_host(hostname, use_cachebrowser_db=True): # Check if host exists in database if use_cachebrowser_db: try: host = Host.get(hostname=hostname) cdn = host.cdn if cdn is None: _bootstrap_host(host) if not cdn.edge_server: _bootstrap_cdn(cdn) return cdn.edge_server, True except Host.DoesNotExist: pass return _dns_request(hostname), False
def resolve_host(hostname, use_cachebrowser_db=True): # Check if host exists in database if use_cachebrowser_db: try: host = Host.get(hostname) cdn = host.cdn if cdn is None: _bootstrap_host(host) addresses = cdn.addresses if addresses is None or len(addresses) == 0: _bootstrap_cdn(cdn) return cdn.addresses[0], True # make it random? except Host.DoesNotExist: pass return _dns_request(hostname), False
def test_database(self): initialize_database(":memory:", reset=True) cdn1 = CDN.create(id="cdn1", name="Sample CDN", edge_server="1.2.3.4") cdn2 = CDN.create(id="cdn2", name="Sample CDN2", edge_server="1.2.3.5") host1 = Host.create(hostname="first.host", cdn=cdn1) host2 = Host.create(hostname="second.host", cdn=cdn2) host3 = Host.create(hostname="third.host", cdn=cdn1) self.assertEqual([host1, host2, host3], list(Host.select())) self.assertEqual([host1, host3], Host.select().where(Host.cdn == cdn1)) self.assertEqual([host1, host3], Host.select().join(CDN).where(CDN.id == 'cdn1')) self.assertEqual(host2, Host.get(Host.cdn == cdn2)) self.assertEqual([], CDN.select().where(CDN.id == "doesntexist")) with self.assertRaises(DoesNotExist): CDN.get(CDN.id == "doesntexist")
def _start_remote(self): http_request = self.request_builder.http_request url = http_request.path parsed_url = urlparse.urlparse(url) try: host = Host.get(url=parsed_url.hostname) if host.ssl: url = url.replace('http', 'https') self.cachebrowsed = True except Host.DoesNotExist: pass logging.info("[%s] %s %s" % (http_request.method, url, '<CACHEBROWSED>' if self.cachebrowsed else '')) request = http_request.get_raw() # request = re.sub(r'^(GET|POST|PUT|DELETE|HEAD) http[s]?://[^/]+/(.+) (\w+)', r'\1 /\2 \3', request) response = http.request(url, raw_request=request) self._connection.start_remote(response)
def _start_remote(self): http_request = self.request_builder.http_request url = http_request.path parsed_url = urlparse.urlparse(url) try: host = Host.get(Host.hostname==parsed_url.hostname) if host.uses_ssl: url = url.replace('http', 'https') self.cachebrowsed = True except Host.DoesNotExist: pass logger.info("[%s] %s %s" % (http_request.method, url, '<CACHEBROWSED>' if self.cachebrowsed else '')) request = http_request.get_raw() # request = re.sub(r'^(GET|POST|PUT|DELETE|HEAD) http[s]?://[^/]+/(.+) (\w+)', r'\1 /\2 \3', request) response = http.request(url, raw_request=request) self._connection.start_remote(response)
def _get_or_bootstrap_host(self, hostname): try: return Host.get(Host.hostname == hostname) except DoesNotExist: try: host_data = self.bootstrapper.lookup_host(hostname) except BootstrapError: raise DoesNotExist host = Host(**host_data) try: host.cdn = self._get_or_bootstrap_cdn(host.cdn_id) except DoesNotExist: host.cdn = CDN.create(id=host.cdn_id, valid=False) host.save(force_insert=True) return host
def request(self, flow): flow._id = self._id_counter self._id_counter += 1 flow.request.scheme_upgraded = False flow.request.headers["host"] = flow.request.pretty_host if flow.request.scheme == "http": try: host = Host.get_or_bootstrap(hostname=flow.request.host) if host.is_active and host.ssl: flow.request.scheme = "https" flow.request.port = 443 flow.request.scheme_upgraded = True except DoesNotExist: pass self.publish_flow(flow) return flow
def delete_host(context, request): hostname = request.params.get('host', '') Host.delete().where(Host.hostname == hostname).execute() request.reply()
def test_unavailable_host__host_in_db__unavailable_cdn__cdn_in_db_with_no_edge(self): cdn = CDN.create(id="missingcdn") Host.create(hostname="host.db", cdn=cdn) with self.assertRaises(CDNNotAvailableError): self.bootstrapper.bootstrap('missingcdn.host')
def test_unavailable_host__host_in_db__available_cdn__cdn_in_db_with_no_edge(self): cdn = CDN.create(id="thecdn") Host.create(hostname="host.db", cdn=cdn) self.bootstrapper.bootstrap('host.db')
def test_bootstrap(self): self.bootstrapper.bootstrap('the.host') self.assertTrue(Host.select().where(Host.hostname == 'the.host').exists()) self.assertTrue(CDN.select().where(CDN.id == 'thecdn').exists())