def _get_or_create_host(cls, hostname, create=True):
try:
return Host.get(Host.hostname == hostname), False
except DoesNotExist:
if create:
return Host.create(hostname=hostname), True
return None, None
def _start_upstream(self):
self._buffer.seek(0)
firstline = self._buffer.readline()
match = re.match("(?:CONNECT) ([^:]+)(?:[:](\d+))? \w+", firstline)
if match is None:
return
host = match.group(1)
port = int(match.group(2) or 443)
cachebrowsed = False
try:
Host.get(Host.hostname == host)
cachebrowsed = True
except Host.DoesNotExist:
pass
if cachebrowsed:
#logger.info("[HTTPS] %s:%s <REJECTED>" % (host, port))
#self.connection.close_local()
logger.info("[HTTPS] %s:%s <CACHEBROWSED>" % (host, port))
return self._connect_upstream(host, port)
else:
logger.info("[HTTPS] %s:%s <PROXYING>" % (host, port))
return self._connect_upstream(host, port)
def _start_upstream(self):
self._buffer.seek(0)
firstline = self._buffer.readline()
match = re.match("(?:CONNECT) ([^:]+)(?:[:](\d+))? \w+", firstline)
if match is None:
return
host = match.group(1)
port = int(match.group(2) or 443)
cachebrowsed = False
try:
Host.get(Host.hostname == host)
cachebrowsed = True
except Host.DoesNotExist:
pass
if cachebrowsed:
#logger.info("[HTTPS] %s:%s <REJECTED>" % (host, port))
#self.connection.close_local()
logger.info("[HTTPS] %s:%s <CACHEBROWSED>" % (host, port))
return self._connect_upstream(host, port)
else:
logger.info("[HTTPS] %s:%s <PROXYING>" % (host, port))
return self._connect_upstream(host, port)
def _get_or_create_host(cls, hostname, create=True):
try:
return Host.get(Host.hostname == hostname), False
except DoesNotExist:
if create:
return Host.create(hostname=hostname), True
return None, None
def get_hosts(context, request):
page = request.params.get('page', 0)
num_per_page = request.params.get('num_per_page', 5)
if num_per_page <= 0:
result = Host.select()
else:
result = Host.select().paginate(page, num_per_page)
hosts = [serialize_host(host) for host in result]
request.reply(hosts)
def get_hosts(context, request):
page = request.params.get('page', 0)
num_per_page = request.params.get('num_per_page', 5)
if num_per_page <= 0:
result = Host.select()
else:
result = Host.select().paginate(page, num_per_page)
hosts = [serialize_host(host) for host in result]
request.reply(hosts)
def bootstrap(context, save, hostname):
try:
host_data = context.bootstrapper.lookup_host(hostname)
except BootstrapError:
logger.warning("No bootstrap information found for host '{}'".format(hostname))
return
logger.info(json.dumps(host_data, indent=4))
if save:
host = Host(**host_data)
host.save()
def serverconnect(self, server_conn):
hostname = server_conn.address.host
# TODO (NOT FOR ALL SITE, BUT NEEDED FOR YOUTUBE)
server_conn.sni = ""
server_conn.cachebrowsed = False
server_conn.cdn = None
server_conn.cb_status_message = ""
try:
host = Host.get_or_bootstrap(hostname=hostname)
except DoesNotExist:
err = "Bootstrapping host {} failed".format(hostname)
server_conn.cb_status_message = err
return server_conn
# Skip if host is not active
if not host.is_active:
server_conn.status = "Host not active"
return server_conn
cdn = host.cdn
if not cdn.valid or not cdn.edge_server:
err = "Host {} does not have a valid CDN".format(hostname)
server_conn.cb_status_message = err
return server_conn
server_conn.address = Address((cdn.edge_server, server_conn.address.port))
server_conn.sni = ""
server_conn.cachebrowsed = True
server_conn.cdn = {"id": cdn.id, "name": cdn.name}
return server_conn
def domain_list(self):
"""
List the active hosts
"""
hosts = Host.select()
for host in hosts:
self.send_line("%*s: %s" % (20, host.url, host.cdn.id))
def domain_list(self):
"""
List the active hosts
"""
hosts = Host.select()
for host in hosts:
self.send_line("%*s: %s" % (20, host.hostname, host.cdn.id))
def get_hosts(request):
page = request.params.get('page', 0)
num_per_page = request.params.get('num_per_page', 5)
result = Host.select().paginate(page, num_per_page) # .order_by(Host.hostname)
hosts = [serialize_host(host) for host in result]
request.reply(hosts)
def add_host(context, request):
host = Host()
host.hostname = request.params.get('hostname', '')
host.cdn = request.params.get('cdn', '')
host.ssl = request.params.get('ssl', True)
host.save(force_insert=True)
request.reply()
def action_check_host(request):
hostname = extract_url_hostname(request['host'])
try:
is_active = Host.get(Host.hostname == hostname).is_active
except DoesNotExist:
is_active = False
return {
'result': 'active' if is_active else 'inactive',
'host': request['host']
}
def add_host(context, request):
host = Host()
host.hostname = request.params.get('hostname', '')
host.cdn = request.params.get('cdn', '')
host.ssl = request.params.get('ssl', True)
host.save(force_insert=True)
request.reply()
def resolve_host(hostname, use_cachebrowser_db=True):
# Check if host exists in database
if use_cachebrowser_db:
try:
host = Host.get(hostname=hostname)
cdn = host.cdn
if cdn is None:
_bootstrap_host(host)
if not cdn.edge_server:
_bootstrap_cdn(cdn)
return cdn.edge_server, True
except Host.DoesNotExist:
pass
return _dns_request(hostname), False
def resolve_host(hostname, use_cachebrowser_db=True):
# Check if host exists in database
if use_cachebrowser_db:
try:
host = Host.get(hostname)
cdn = host.cdn
if cdn is None:
_bootstrap_host(host)
addresses = cdn.addresses
if addresses is None or len(addresses) == 0:
_bootstrap_cdn(cdn)
return cdn.addresses[0], True # make it random?
except Host.DoesNotExist:
pass
return _dns_request(hostname), False
def test_database(self):
initialize_database(":memory:", reset=True)
cdn1 = CDN.create(id="cdn1", name="Sample CDN", edge_server="1.2.3.4")
cdn2 = CDN.create(id="cdn2", name="Sample CDN2", edge_server="1.2.3.5")
host1 = Host.create(hostname="first.host", cdn=cdn1)
host2 = Host.create(hostname="second.host", cdn=cdn2)
host3 = Host.create(hostname="third.host", cdn=cdn1)
self.assertEqual([host1, host2, host3], list(Host.select()))
self.assertEqual([host1, host3], Host.select().where(Host.cdn == cdn1))
self.assertEqual([host1, host3], Host.select().join(CDN).where(CDN.id == 'cdn1'))
self.assertEqual(host2, Host.get(Host.cdn == cdn2))
self.assertEqual([], CDN.select().where(CDN.id == "doesntexist"))
with self.assertRaises(DoesNotExist):
CDN.get(CDN.id == "doesntexist")
def _start_remote(self):
http_request = self.request_builder.http_request
url = http_request.path
parsed_url = urlparse.urlparse(url)
try:
host = Host.get(url=parsed_url.hostname)
if host.ssl:
url = url.replace('http', 'https')
self.cachebrowsed = True
except Host.DoesNotExist:
pass
logging.info("[%s] %s %s" % (http_request.method, url, '<CACHEBROWSED>' if self.cachebrowsed else ''))
request = http_request.get_raw()
# request = re.sub(r'^(GET|POST|PUT|DELETE|HEAD) http[s]?://[^/]+/(.+) (\w+)', r'\1 /\2 \3', request)
response = http.request(url, raw_request=request)
self._connection.start_remote(response)
def _start_remote(self):
http_request = self.request_builder.http_request
url = http_request.path
parsed_url = urlparse.urlparse(url)
try:
host = Host.get(Host.hostname==parsed_url.hostname)
if host.uses_ssl:
url = url.replace('http', 'https')
self.cachebrowsed = True
except Host.DoesNotExist:
pass
logger.info("[%s] %s %s" % (http_request.method, url, '<CACHEBROWSED>' if self.cachebrowsed else ''))
request = http_request.get_raw()
# request = re.sub(r'^(GET|POST|PUT|DELETE|HEAD) http[s]?://[^/]+/(.+) (\w+)', r'\1 /\2 \3', request)
response = http.request(url, raw_request=request)
self._connection.start_remote(response)
def _get_or_bootstrap_host(self, hostname):
try:
return Host.get(Host.hostname == hostname)
except DoesNotExist:
try:
host_data = self.bootstrapper.lookup_host(hostname)
except BootstrapError:
raise DoesNotExist
host = Host(**host_data)
try:
host.cdn = self._get_or_bootstrap_cdn(host.cdn_id)
except DoesNotExist:
host.cdn = CDN.create(id=host.cdn_id, valid=False)
host.save(force_insert=True)
return host
def request(self, flow):
flow._id = self._id_counter
self._id_counter += 1
flow.request.scheme_upgraded = False
flow.request.headers["host"] = flow.request.pretty_host
if flow.request.scheme == "http":
try:
host = Host.get_or_bootstrap(hostname=flow.request.host)
if host.is_active and host.ssl:
flow.request.scheme = "https"
flow.request.port = 443
flow.request.scheme_upgraded = True
except DoesNotExist:
pass
self.publish_flow(flow)
return flow
def delete_host(context, request):
hostname = request.params.get('host', '')
Host.delete().where(Host.hostname == hostname).execute()
request.reply()
def delete_host(context, request):
hostname = request.params.get('host', '')
Host.delete().where(Host.hostname == hostname).execute()
request.reply()
def test_unavailable_host__host_in_db__unavailable_cdn__cdn_in_db_with_no_edge(self):
cdn = CDN.create(id="missingcdn")
Host.create(hostname="host.db", cdn=cdn)
with self.assertRaises(CDNNotAvailableError):
self.bootstrapper.bootstrap('missingcdn.host')
def test_unavailable_host__host_in_db__available_cdn__cdn_in_db_with_no_edge(self):
cdn = CDN.create(id="thecdn")
Host.create(hostname="host.db", cdn=cdn)
self.bootstrapper.bootstrap('host.db')
def test_bootstrap(self):
self.bootstrapper.bootstrap('the.host')
self.assertTrue(Host.select().where(Host.hostname == 'the.host').exists())
self.assertTrue(CDN.select().where(CDN.id == 'thecdn').exists())
