Esempio n. 1
0
def gae_urlfetch(client, proxy, method, url, headers, payload, **kwargs):
    if payload:
        if len(payload) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
            zpayload = zlib.compress(payload)[2:-4]
            if len(zpayload) < len(payload):
                payload = zpayload
                headers['Content-Encoding'] = 'deflate'
        headers['Content-Length'] = str(len(payload))
        # GAE donot allow set `Host` header
    if 'Host' in headers:
        del headers['Host']
    metadata = 'G-Method:%s\nG-Url:%s\n%s' % (
    method, url, ''.join('G-%s:%s\n' % (k, v) for k, v in kwargs.items() if v))
    metadata += ''.join('%s:%s\n' % (k.title(), v) for k, v in headers.items() if k not in SKIP_HEADERS)
    metadata = zlib.compress(metadata.encode())[2:-4]
    payload = b''.join((struct.pack('!h', len(metadata)), metadata, payload))
    ssl_sock = create_ssl_connection()
    ssl_sock.counter = stat.opened(ssl_sock, proxy, host=client.host, ip=client.dst_ip)
    LOGGER.info('[%s] urlfetch %s %s via %s %0.2f'
                % (repr(client), method, url, ssl_sock.google_ip, get_google_ip_latency(ssl_sock.google_ip)))
    client.add_resource(ssl_sock)
    client.add_resource(ssl_sock.counter)
    client.add_resource(ssl_sock.sock)
    response = http_call(ssl_sock, 'POST', proxy.fetch_server, {'Content-Length': str(len(payload))}, payload)
    client.add_resource(response.rfile)
    client.add_resource(response.counted_sock)
    response.app_status = response.status
    if response.status != 200:
        return response
    data = response.read(4)
    if len(data) < 4:
        response.status = 502
        response.fp = io.BytesIO(b'connection aborted. too short leadtype data=' + data)
        return response
    response.status, headers_length = struct.unpack('!hh', data)
    data = response.read(headers_length)
    if len(data) < headers_length:
        response.status = 502
        response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)
        return response
    response.headers = response.msg = http.client.parse_headers(io.BytesIO(zlib.decompress(data, -zlib.MAX_WBITS)))
    return response
Esempio n. 2
0
def gae_urlfetch_for_lower_than_3_2_0(client, proxy, method, url, headers, payload):
    password=proxy.password
    is_obfuscate_enabled=proxy.is_obfuscate_enabled
    is_rc4_enabled=proxy.is_rc4_enabled
    if payload:
        if len(payload) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
            zpayload = zlib.compress(payload)[2:-4]
            if len(zpayload) < len(payload):
                payload = zpayload
                headers['Content-Encoding'] = 'deflate'
        headers['Content-Length'] = str(len(payload))
        # GAE donot allow set `Host` header
    if 'Host' in headers:
        del headers['Host']
    metadata = 'G-Method:%s\nG-Url:%s\n' % (method, url)
    if password:
        metadata += 'G-password:%s\n' % password
    metadata += ''.join('%s:%s\n' % (k.title(), v) for k, v in headers.items() if k not in SKIP_HEADERS)
    metadata = metadata.encode('utf8')
    request_method = 'POST'
    request_headers = {}
    if is_obfuscate_enabled:
        if is_rc4_enabled:
            request_headers['X-GOA-Options'] = 'rc4'
            cookie = base64.b64encode(rc4crypt(zlib.compress(metadata)[2:-4], password)).strip()
            payload = rc4crypt(payload, password)
        else:
            cookie = base64.b64encode(zlib.compress(metadata)[2:-4]).strip()
        request_headers['Cookie'] = cookie
        if payload:
            request_headers['Content-Length'] = str(len(payload))
        else:
            request_method = 'GET'
    else:
        metadata = zlib.compress(metadata)[2:-4]
        payload = '%s%s%s' % (struct.pack('!h', len(metadata)), metadata, payload)
        if is_rc4_enabled:
            request_headers['X-GOA-Options'] = 'rc4'
            payload = rc4crypt(payload, password)
        request_headers['Content-Length'] = str(len(payload))
    ssl_sock = create_ssl_connection()
    ssl_sock.counter = stat.opened(ssl_sock, proxy, host=client.host, ip=client.dst_ip)
    LOGGER.info('[%s] urlfetch %s %s via %s %0.2f'
                % (repr(client), method, url, ssl_sock.google_ip, get_google_ip_latency(ssl_sock.google_ip)))
    client.add_resource(ssl_sock)
    client.add_resource(ssl_sock.counter)
    client.add_resource(ssl_sock.sock)
    response = http_call(ssl_sock, request_method, proxy.fetch_server, request_headers, payload)
    client.add_resource(response.rfile)
    client.add_resource(response.counted_sock)
    response.app_status = response.status
    response.app_options = response.getheader('X-GOA-Options', '')
    if response.status != 200:
        return response
    data = response.read(4)
    if len(data) < 4:
        response.status = 502
        response.fp = io.BytesIO(b'connection aborted. too short leadtype data=' + data)
        response.read = response.fp.read
        return response
    response.status, headers_length = struct.unpack('!hh', data)
    data = response.read(headers_length)
    if len(data) < headers_length:
        response.status = 502
        response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)
        response.read = response.fp.read
        return response
    if 'rc4' not in response.app_options:
        response.msg = httplib.HTTPMessage(io.BytesIO(zlib.decompress(data, -zlib.MAX_WBITS)))
    else:
        response.msg = httplib.HTTPMessage(io.BytesIO(zlib.decompress(rc4crypt(data, password), -zlib.MAX_WBITS)))
        if password and response.fp:
            response.fp = RC4FileObject(response.fp, password)
    return response
Esempio n. 3
0
def gae_urlfetch_for_3_2_0_or_above(client, proxy, method, url, headers, body):

    def inflate(data):
        return zlib.decompress(data, -zlib.MAX_WBITS)

    def deflate(data):
        return zlib.compress(data)[2:-4]

    password=proxy.password
    is_obfuscate_enabled=proxy.is_obfuscate_enabled
    is_rc4_enabled=proxy.is_rc4_enabled

    if isinstance(body, basestring) and body:
        if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
            zbody = deflate(body)
            if len(zbody) < len(body):
                body = zbody
                headers['Content-Encoding'] = 'deflate'
        headers['Content-Length'] = str(len(body))
    # GAE donot allow set `Host` header
    if 'Host' in headers:
        del headers['Host']
    kwargs = {}
    if password:
        kwargs['password'] = password
    payload = '%s %s %s\r\n' % (method, url, 'HTTP/1.0')
    payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in SKIP_HEADERS)
    payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v) for k, v in kwargs.items() if v)
    # prepare GAE request
    request_method = 'POST'
    request_headers = {}
    fetch_server = proxy.fetch_server
    if is_obfuscate_enabled:
        request_method = 'GET'
        fetch_server += 'ps/%d%s.gif' % (int(time.time()*1000), random.random())
        request_headers['X-URLFETCH-PS1'] = base64.b64encode(deflate(payload)).strip()
        if body:
            request_headers['X-URLFETCH-PS2'] = base64.b64encode(deflate(body)).strip()
            body = ''
    else:
        payload = deflate(payload)
        body = '%s%s%s' % (struct.pack('!h', len(payload)), payload, body)
        if is_rc4_enabled:
            request_headers['X-URLFETCH-Options'] = 'rc4'
            body = rc4crypt(body, password)
        request_headers['Content-Length'] = str(len(body))
    # post data
    ssl_sock = create_ssl_connection()
    ssl_sock.counter = stat.opened(ssl_sock, proxy, host=client.host, ip=client.dst_ip)
    LOGGER.info('[%s] urlfetch %s %s via %s %0.2f'
                % (repr(client), method, url, ssl_sock.google_ip, get_google_ip_latency(ssl_sock.google_ip)))
    client.add_resource(ssl_sock)
    client.add_resource(ssl_sock.counter)
    client.add_resource(ssl_sock.sock)
    response = http_call(ssl_sock, request_method, fetch_server, request_headers, body)
    client.add_resource(response.rfile)
    client.add_resource(response.counted_sock)
    response.app_status = response.status
    if response.status != 200:
        return response
    if is_rc4_enabled:
        response.fp = RC4FileObject(response.fp, password)
    data = response.read(2)
    if len(data) < 2:
        response.status = 502
        response.fp = io.BytesIO(b'connection aborted. too short leadbyte data=' + data)
        response.read = response.fp.read
        return response
    headers_length, = struct.unpack('!h', data)
    data = response.read(headers_length)
    if len(data) < headers_length:
        response.status = 502
        response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)
        response.read = response.fp.read
        return response
    raw_response_line, headers_data = inflate(data).split('\r\n', 1)
    _, response.status, response.reason = raw_response_line.split(None, 2)
    response.status = int(response.status)
    response.reason = response.reason.strip()
    response.msg = httplib.HTTPMessage(io.BytesIO(headers_data))
    return response