Example #1
0
 def do_forward(self, client):
     try:
         if not recv_and_parse_request(client):
             raise Exception('payload is too large')
         if client.method.upper() not in ('GET', 'POST', 'HEAD'):
             raise Exception('unsupported method: %s' % client.method)
         if client.host in GoAgentProxy.black_list:
             raise Exception('%s failed to proxy via goagent before' % client.host)
     except NotHttp:
         raise
     except:
         for proxy in self.proxies:
             client.tried_proxies[proxy] = 'skip goagent'
         LOGGER.error('[%s] failed to recv and parse request: %s' % (repr(client), sys.exc_info()[1]))
         client.fall_back(reason='failed to recv and parse request, %s' % sys.exc_info()[1])
     forward(client, self)
Example #2
0
 def do_forward(self, client):
     try:
         if not recv_and_parse_request(client):
             raise Exception('payload is too large')
         if client.method.upper() not in ('GET', 'POST', 'HEAD'):
             raise Exception('unsupported method: %s' % client.method)
         if client.host in GoAgentProxy.global_black_list:
             raise Exception('%s failed to proxy via goagent before' % client.host)
     except NotHttp:
         raise
     except:
         for proxy in self.proxies:
             client.tried_proxies[proxy] = 'skip goagent'
         LOGGER.error('[%s] failed to recv and parse request: %s' % (repr(client), sys.exc_info()[1]))
         client.fall_back(reason='failed to recv and parse request, %s' % sys.exc_info()[1])
     forward(client, self)
Example #3
0
def forward(client, proxy):
    parsed_url = urllib.parse.urlparse(client.url)
    range_in_query = 'range=' in parsed_url.query or 'redirect_counter=' in parsed_url.query
    special_range = (any(x(client.host) for x in AUTORANGE_HOSTS_MATCH) or client.url.endswith(
        AUTORANGE_ENDSWITH)) and not client.url.endswith(
        AUTORANGE_NOENDSWITH) and not 'redirector.c.youtube.com' == client.host
    if client.host in GoAgentProxy.gray_list:
        special_range = True
    range_end = 0
    auto_ranged = False
    if 'Range' in client.headers:
        LOGGER.info('[%s] range present: %s' % (repr(client), client.headers['Range']))
        m = re.search('bytes=(\d+)-(\d*)', client.headers['Range'])
        if m:
            range_start = int(m.group(1))
            range_end = int(m.group(2)) if m.group(2) else 0
            if not range_end or range_end - range_start > AUTORANGE_MAXSIZE:
                client.headers['Range'] = 'bytes=%d-%d' % (range_start, range_start + AUTORANGE_MAXSIZE)
                LOGGER.info('[%s] adjusted range: %s' % (repr(client), client.headers['Range']))
    elif not range_in_query and special_range:
        client.headers['Range'] = 'bytes=%d-%d' % (0, AUTORANGE_MAXSIZE)
        auto_ranged = True
        LOGGER.info('[%s] auto range: %s' % (repr(client), client.headers['Range']))
    response = None
    try:
        kwargs = {}
        if proxy.password:
            kwargs['password'] = proxy.password
        try:
            response = gae_urlfetch(
                client, proxy, client.method, client.url, client.headers, client.payload, **kwargs)
        except ConnectionFailed:
            for proxy in GoAgentProxy.proxies:
                client.tried_proxies[proxy] = 'skip goagent'
            client.fall_back('can not connect to google ip')
        except ReadResponseFailed:
            if 'youtube.com' not in client.host and 'googlevideo.com' not in client.host:
                GoAgentProxy.gray_list.add(client.host)
                if auto_ranged:
                    LOGGER.error('[%s] !!! blacklist goagent for %s !!!' % (repr(client), client.host))
                    GoAgentProxy.black_list.add(client.host)
                    if client.host in HttpTryProxy.host_slow_list:
                        HttpTryProxy.host_slow_list.remove(client.host)
            for proxy in GoAgentProxy.proxies:
                client.tried_proxies[proxy] = 'skip goagent'
            client.fall_back(reason='failed to read response from gae_urlfetch')
        if response is None:
            client.fall_back('urlfetch empty response')
        if response.app_status == 503:
            proxy.died = True
            if time.time() - GoAgentProxy.last_refresh_started_at > 60:
                GoAgentProxy.last_refresh_started_at = time.time()
                LOGGER.error('refresh goagent proxies due to over quota')
                gevent.spawn(GoAgentProxy.refresh, GoAgentProxy.proxies)
            client.fall_back('goagent server over quota')
        if response.app_status == 500:
            proxy.died = True
            client.fall_back('goagent server busy')
        if response.app_status == 404:
            proxy.died = True
            client.fall_back('goagent server not found')
        if response.app_status == 302:
            proxy.died = True
            client.fall_back('goagent server 302 moved')
        if response.app_status == 403 and 'youtube.com' in client.url:
            proxy.died = True
            client.fall_back('goagent server %s banned youtube' % proxy)
        if response.app_status != 200:
            if LOGGER.isEnabledFor(logging.DEBUG):
                LOGGER.debug('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
                    '%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
                LOGGER.debug(response.read())
            client.fall_back('urlfetch failed: %s' % response.app_status)
        client.forward_started = True
        if response.status == 206:
            LOGGER.info('[%s] start range fetch' % repr(client))
            rangefetch = RangeFetch(client, range_end, auto_ranged, response)
            return rangefetch.fetch()
        if 'Set-Cookie' in response.msg:
            response.msg['Set-Cookie'] = normcookie(response.msg['Set-Cookie'])
        client.downstream_wfile.write('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
            '%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
        content_length = int(response.getheader('Content-Length', 0))
        content_range = response.getheader('Content-Range', '')
        if content_range:
            start, end, length = list(map(int, re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)))
        else:
            start, end, length = 0, content_length - 1, content_length
        while 1:
            try:
                data = response.read(8192)
                response.ssl_sock.counter.received(len(response.counted_sock.rfile.captured))
                response.counted_sock.rfile.captured = ''
            except httplib.IncompleteRead as e:
                LOGGER.error('incomplete read: %s' % e.partial)
                raise
            if not data:
                response.close()
                return
            start += len(data)
            client.downstream_wfile.write(data)
            if start >= end:
                response.close()
                return
    finally:
        if response:
            response.close()
Example #4
0
def forward(client, proxy):
    parsed_url = urllib.parse.urlparse(client.url)
    range_in_query = 'range=' in parsed_url.query or 'redirect_counter=' in parsed_url.query
    special_range = (any(x(client.host) for x in AUTORANGE_HOSTS_MATCH) or client.url.endswith(
        AUTORANGE_ENDSWITH)) and not client.url.endswith(
        AUTORANGE_NOENDSWITH) and not 'redirector.c.youtube.com' == client.host
    if client.host in GoAgentProxy.global_gray_list:
        special_range = True
    range_end = 0
    auto_ranged = False
    if 'Range' in client.headers:
        LOGGER.info('[%s] range present: %s' % (repr(client), client.headers['Range']))
        m = re.search('bytes=(\d+)-(\d*)', client.headers['Range'])
        if m:
            range_start = int(m.group(1))
            range_end = int(m.group(2)) if m.group(2) else 0
            if not range_end or range_end - range_start > AUTORANGE_MAXSIZE:
                client.headers['Range'] = 'bytes=%d-%d' % (range_start, range_start + AUTORANGE_MAXSIZE)
                LOGGER.info('[%s] adjusted range: %s' % (repr(client), client.headers['Range']))
    elif not range_in_query and special_range:
        client.headers['Range'] = 'bytes=%d-%d' % (0, AUTORANGE_MAXSIZE)
        auto_ranged = True
        LOGGER.info('[%s] auto range: %s' % (repr(client), client.headers['Range']))
    response = None
    try:
        try:
            response = proxy.gae_urlfetch(
                client, proxy, client.method, client.url, client.headers, client.payload)
        except ConnectionFailed:
            for proxy in GoAgentProxy.proxies:
                client.tried_proxies[proxy] = 'skip goagent'
            client.fall_back('can not connect to google ip')
        except ReadResponseFailed:
            if 'youtube.com' not in client.host and 'googlevideo.com' not in client.host:
                GoAgentProxy.global_gray_list.add(client.host)
                if auto_ranged:
                    LOGGER.error('[%s] !!! blacklist goagent for %s !!!' % (repr(client), client.host))
                    GoAgentProxy.global_black_list.add(client.host)
                    if client.host in HttpTryProxy.host_slow_list:
                        HttpTryProxy.host_slow_list.remove(client.host)
            for proxy in GoAgentProxy.proxies:
                client.tried_proxies[proxy] = 'skip goagent'
            client.fall_back(reason='failed to read response from gae_urlfetch')
        if response is None:
            client.fall_back('urlfetch empty response')
        if response.app_status == 503:
            LOGGER.error('%s died due to 503' % proxy)
            proxy.died = True
            client.fall_back('goagent server over quota')
        if response.app_status == 500:
            # LOGGER.error('%s died due to 500' % proxy)
            # proxy.died = True
            client.fall_back('goagent server busy')
        if response.app_status == 404:
            LOGGER.error('%s died due to 404' % proxy)
            proxy.died = True
            client.fall_back('goagent server not found')
        if response.app_status == 302:
            LOGGER.error('%s died due to 302' % proxy)
            proxy.died = True
            client.fall_back('goagent server 302 moved')
        if response.app_status == 403:
            # LOGGER.error('%s died due to 403' % proxy)
            # proxy.died = True
            client.fall_back('goagent server %s banned this host' % proxy)
        if response.app_status != 200:
            if LOGGER.isEnabledFor(logging.DEBUG):
                LOGGER.debug('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
                    '%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
                LOGGER.debug(response.read())
            client.fall_back('urlfetch failed: %s' % response.app_status)
        client.forward_started = True
        if response.status == 206:
            LOGGER.info('[%s] start range fetch' % repr(client))
            rangefetch = RangeFetch(client, range_end, auto_ranged, response)
            return rangefetch.fetch()
        if 'Set-Cookie' in response.msg:
            response.msg['Set-Cookie'] = normcookie(response.msg['Set-Cookie'])
        client.downstream_wfile.write('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
            '%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k != 'transfer-encoding')))
        content_length = int(response.getheader('Content-Length', 0))
        content_range = response.getheader('Content-Range', '')
        if content_range:
            start, end, length = list(map(int, re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)))
        else:
            start, end, length = 0, content_length - 1, content_length
        while 1:
            try:
                data = response.read(8192)
                response.ssl_sock.counter.received(len(response.counted_sock.rfile.captured))
                response.counted_sock.rfile.captured = ''
            except httplib.IncompleteRead as e:
                LOGGER.error('incomplete read: %s' % e.partial)
                raise
            if not data:
                response.close()
                return
            start += len(data)
            client.downstream_wfile.write(data)
            if start >= end:
                response.close()
                return
    finally:
        if response:
            response.close()