def do_forward(self, client):
try:
if not recv_and_parse_request(client):
raise Exception('payload is too large')
if client.method.upper() not in ('GET', 'POST', 'HEAD'):
raise Exception('unsupported method: %s' % client.method)
if client.host in GoAgentProxy.black_list:
raise Exception('%s failed to proxy via goagent before' % client.host)
except NotHttp:
raise
except:
for proxy in self.proxies:
client.tried_proxies[proxy] = 'skip goagent'
LOGGER.error('[%s] failed to recv and parse request: %s' % (repr(client), sys.exc_info()[1]))
client.fall_back(reason='failed to recv and parse request, %s' % sys.exc_info()[1])
forward(client, self)
def do_forward(self, client):
try:
if not recv_and_parse_request(client):
raise Exception('payload is too large')
if client.method.upper() not in ('GET', 'POST', 'HEAD'):
raise Exception('unsupported method: %s' % client.method)
if client.host in GoAgentProxy.global_black_list:
raise Exception('%s failed to proxy via goagent before' % client.host)
except NotHttp:
raise
except:
for proxy in self.proxies:
client.tried_proxies[proxy] = 'skip goagent'
LOGGER.error('[%s] failed to recv and parse request: %s' % (repr(client), sys.exc_info()[1]))
client.fall_back(reason='failed to recv and parse request, %s' % sys.exc_info()[1])
forward(client, self)
def forward(client, proxy):
parsed_url = urllib.parse.urlparse(client.url)
range_in_query = 'range=' in parsed_url.query or 'redirect_counter=' in parsed_url.query
special_range = (any(x(client.host) for x in AUTORANGE_HOSTS_MATCH) or client.url.endswith(
AUTORANGE_ENDSWITH)) and not client.url.endswith(
AUTORANGE_NOENDSWITH) and not 'redirector.c.youtube.com' == client.host
if client.host in GoAgentProxy.gray_list:
special_range = True
range_end = 0
auto_ranged = False
if 'Range' in client.headers:
LOGGER.info('[%s] range present: %s' % (repr(client), client.headers['Range']))
m = re.search('bytes=(\d+)-(\d*)', client.headers['Range'])
if m:
range_start = int(m.group(1))
range_end = int(m.group(2)) if m.group(2) else 0
if not range_end or range_end - range_start > AUTORANGE_MAXSIZE:
client.headers['Range'] = 'bytes=%d-%d' % (range_start, range_start + AUTORANGE_MAXSIZE)
LOGGER.info('[%s] adjusted range: %s' % (repr(client), client.headers['Range']))
elif not range_in_query and special_range:
client.headers['Range'] = 'bytes=%d-%d' % (0, AUTORANGE_MAXSIZE)
auto_ranged = True
LOGGER.info('[%s] auto range: %s' % (repr(client), client.headers['Range']))
response = None
try:
kwargs = {}
if proxy.password:
kwargs['password'] = proxy.password
try:
response = gae_urlfetch(
client, proxy, client.method, client.url, client.headers, client.payload, **kwargs)
except ConnectionFailed:
for proxy in GoAgentProxy.proxies:
client.tried_proxies[proxy] = 'skip goagent'
client.fall_back('can not connect to google ip')
except ReadResponseFailed:
if 'youtube.com' not in client.host and 'googlevideo.com' not in client.host:
GoAgentProxy.gray_list.add(client.host)
if auto_ranged:
LOGGER.error('[%s] !!! blacklist goagent for %s !!!' % (repr(client), client.host))
GoAgentProxy.black_list.add(client.host)
if client.host in HttpTryProxy.host_slow_list:
HttpTryProxy.host_slow_list.remove(client.host)
for proxy in GoAgentProxy.proxies:
client.tried_proxies[proxy] = 'skip goagent'
client.fall_back(reason='failed to read response from gae_urlfetch')
if response is None:
client.fall_back('urlfetch empty response')
if response.app_status == 503:
proxy.died = True
if time.time() - GoAgentProxy.last_refresh_started_at > 60:
GoAgentProxy.last_refresh_started_at = time.time()
LOGGER.error('refresh goagent proxies due to over quota')
gevent.spawn(GoAgentProxy.refresh, GoAgentProxy.proxies)
client.fall_back('goagent server over quota')
if response.app_status == 500:
proxy.died = True
client.fall_back('goagent server busy')
if response.app_status == 404:
proxy.died = True
client.fall_back('goagent server not found')
if response.app_status == 302:
proxy.died = True
client.fall_back('goagent server 302 moved')
if response.app_status == 403 and 'youtube.com' in client.url:
proxy.died = True
client.fall_back('goagent server %s banned youtube' % proxy)
if response.app_status != 200:
if LOGGER.isEnabledFor(logging.DEBUG):
LOGGER.debug('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
'%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
LOGGER.debug(response.read())
client.fall_back('urlfetch failed: %s' % response.app_status)
client.forward_started = True
if response.status == 206:
LOGGER.info('[%s] start range fetch' % repr(client))
rangefetch = RangeFetch(client, range_end, auto_ranged, response)
return rangefetch.fetch()
if 'Set-Cookie' in response.msg:
response.msg['Set-Cookie'] = normcookie(response.msg['Set-Cookie'])
client.downstream_wfile.write('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
'%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = list(map(int, re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)))
else:
start, end, length = 0, content_length - 1, content_length
while 1:
try:
data = response.read(8192)
response.ssl_sock.counter.received(len(response.counted_sock.rfile.captured))
response.counted_sock.rfile.captured = ''
except httplib.IncompleteRead as e:
LOGGER.error('incomplete read: %s' % e.partial)
raise
if not data:
response.close()
return
start += len(data)
client.downstream_wfile.write(data)
if start >= end:
response.close()
return
finally:
if response:
response.close()
def forward(client, proxy):
parsed_url = urllib.parse.urlparse(client.url)
range_in_query = 'range=' in parsed_url.query or 'redirect_counter=' in parsed_url.query
special_range = (any(x(client.host) for x in AUTORANGE_HOSTS_MATCH) or client.url.endswith(
AUTORANGE_ENDSWITH)) and not client.url.endswith(
AUTORANGE_NOENDSWITH) and not 'redirector.c.youtube.com' == client.host
if client.host in GoAgentProxy.global_gray_list:
special_range = True
range_end = 0
auto_ranged = False
if 'Range' in client.headers:
LOGGER.info('[%s] range present: %s' % (repr(client), client.headers['Range']))
m = re.search('bytes=(\d+)-(\d*)', client.headers['Range'])
if m:
range_start = int(m.group(1))
range_end = int(m.group(2)) if m.group(2) else 0
if not range_end or range_end - range_start > AUTORANGE_MAXSIZE:
client.headers['Range'] = 'bytes=%d-%d' % (range_start, range_start + AUTORANGE_MAXSIZE)
LOGGER.info('[%s] adjusted range: %s' % (repr(client), client.headers['Range']))
elif not range_in_query and special_range:
client.headers['Range'] = 'bytes=%d-%d' % (0, AUTORANGE_MAXSIZE)
auto_ranged = True
LOGGER.info('[%s] auto range: %s' % (repr(client), client.headers['Range']))
response = None
try:
try:
response = proxy.gae_urlfetch(
client, proxy, client.method, client.url, client.headers, client.payload)
except ConnectionFailed:
for proxy in GoAgentProxy.proxies:
client.tried_proxies[proxy] = 'skip goagent'
client.fall_back('can not connect to google ip')
except ReadResponseFailed:
if 'youtube.com' not in client.host and 'googlevideo.com' not in client.host:
GoAgentProxy.global_gray_list.add(client.host)
if auto_ranged:
LOGGER.error('[%s] !!! blacklist goagent for %s !!!' % (repr(client), client.host))
GoAgentProxy.global_black_list.add(client.host)
if client.host in HttpTryProxy.host_slow_list:
HttpTryProxy.host_slow_list.remove(client.host)
for proxy in GoAgentProxy.proxies:
client.tried_proxies[proxy] = 'skip goagent'
client.fall_back(reason='failed to read response from gae_urlfetch')
if response is None:
client.fall_back('urlfetch empty response')
if response.app_status == 503:
LOGGER.error('%s died due to 503' % proxy)
proxy.died = True
client.fall_back('goagent server over quota')
if response.app_status == 500:
# LOGGER.error('%s died due to 500' % proxy)
# proxy.died = True
client.fall_back('goagent server busy')
if response.app_status == 404:
LOGGER.error('%s died due to 404' % proxy)
proxy.died = True
client.fall_back('goagent server not found')
if response.app_status == 302:
LOGGER.error('%s died due to 302' % proxy)
proxy.died = True
client.fall_back('goagent server 302 moved')
if response.app_status == 403:
# LOGGER.error('%s died due to 403' % proxy)
# proxy.died = True
client.fall_back('goagent server %s banned this host' % proxy)
if response.app_status != 200:
if LOGGER.isEnabledFor(logging.DEBUG):
LOGGER.debug('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
'%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
LOGGER.debug(response.read())
client.fall_back('urlfetch failed: %s' % response.app_status)
client.forward_started = True
if response.status == 206:
LOGGER.info('[%s] start range fetch' % repr(client))
rangefetch = RangeFetch(client, range_end, auto_ranged, response)
return rangefetch.fetch()
if 'Set-Cookie' in response.msg:
response.msg['Set-Cookie'] = normcookie(response.msg['Set-Cookie'])
client.downstream_wfile.write('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
'%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = list(map(int, re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)))
else:
start, end, length = 0, content_length - 1, content_length
while 1:
try:
data = response.read(8192)
response.ssl_sock.counter.received(len(response.counted_sock.rfile.captured))
response.counted_sock.rfile.captured = ''
except httplib.IncompleteRead as e:
LOGGER.error('incomplete read: %s' % e.partial)
raise
if not data:
response.close()
return
start += len(data)
client.downstream_wfile.write(data)
if start >= end:
response.close()
return
finally:
if response:
response.close()