def gae_urlfetch(client, proxy, method, url, headers, payload, **kwargs):
if payload:
if len(payload) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
zpayload = zlib.compress(payload)[2:-4]
if len(zpayload) < len(payload):
payload = zpayload
headers['Content-Encoding'] = 'deflate'
headers['Content-Length'] = str(len(payload))
# GAE donot allow set `Host` header
if 'Host' in headers:
del headers['Host']
metadata = 'G-Method:%s\nG-Url:%s\n%s' % (
method, url, ''.join('G-%s:%s\n' % (k, v) for k, v in kwargs.items() if v))
metadata += ''.join('%s:%s\n' % (k.title(), v) for k, v in headers.items() if k not in SKIP_HEADERS)
metadata = zlib.compress(metadata.encode())[2:-4]
payload = b''.join((struct.pack('!h', len(metadata)), metadata, payload))
ssl_sock = create_ssl_connection()
ssl_sock.counter = stat.opened(ssl_sock, proxy, host=client.host, ip=client.dst_ip)
LOGGER.info('[%s] urlfetch %s %s via %s %0.2f'
% (repr(client), method, url, ssl_sock.google_ip, get_google_ip_latency(ssl_sock.google_ip)))
client.add_resource(ssl_sock)
client.add_resource(ssl_sock.counter)
client.add_resource(ssl_sock.sock)
response = http_call(ssl_sock, 'POST', proxy.fetch_server, {'Content-Length': str(len(payload))}, payload)
client.add_resource(response.rfile)
client.add_resource(response.counted_sock)
response.app_status = response.status
if response.status != 200:
return response
data = response.read(4)
if len(data) < 4:
response.status = 502
response.fp = io.BytesIO(b'connection aborted. too short leadtype data=' + data)
return response
response.status, headers_length = struct.unpack('!hh', data)
data = response.read(headers_length)
if len(data) < headers_length:
response.status = 502
response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)
return response
response.headers = response.msg = http.client.parse_headers(io.BytesIO(zlib.decompress(data, -zlib.MAX_WBITS)))
return response
def gae_urlfetch_for_lower_than_3_2_0(client, proxy, method, url, headers, payload):
password=proxy.password
is_obfuscate_enabled=proxy.is_obfuscate_enabled
is_rc4_enabled=proxy.is_rc4_enabled
if payload:
if len(payload) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
zpayload = zlib.compress(payload)[2:-4]
if len(zpayload) < len(payload):
payload = zpayload
headers['Content-Encoding'] = 'deflate'
headers['Content-Length'] = str(len(payload))
# GAE donot allow set `Host` header
if 'Host' in headers:
del headers['Host']
metadata = 'G-Method:%s\nG-Url:%s\n' % (method, url)
if password:
metadata += 'G-password:%s\n' % password
metadata += ''.join('%s:%s\n' % (k.title(), v) for k, v in headers.items() if k not in SKIP_HEADERS)
metadata = metadata.encode('utf8')
request_method = 'POST'
request_headers = {}
if is_obfuscate_enabled:
if is_rc4_enabled:
request_headers['X-GOA-Options'] = 'rc4'
cookie = base64.b64encode(rc4crypt(zlib.compress(metadata)[2:-4], password)).strip()
payload = rc4crypt(payload, password)
else:
cookie = base64.b64encode(zlib.compress(metadata)[2:-4]).strip()
request_headers['Cookie'] = cookie
if payload:
request_headers['Content-Length'] = str(len(payload))
else:
request_method = 'GET'
else:
metadata = zlib.compress(metadata)[2:-4]
payload = '%s%s%s' % (struct.pack('!h', len(metadata)), metadata, payload)
if is_rc4_enabled:
request_headers['X-GOA-Options'] = 'rc4'
payload = rc4crypt(payload, password)
request_headers['Content-Length'] = str(len(payload))
ssl_sock = create_ssl_connection()
ssl_sock.counter = stat.opened(ssl_sock, proxy, host=client.host, ip=client.dst_ip)
LOGGER.info('[%s] urlfetch %s %s via %s %0.2f'
% (repr(client), method, url, ssl_sock.google_ip, get_google_ip_latency(ssl_sock.google_ip)))
client.add_resource(ssl_sock)
client.add_resource(ssl_sock.counter)
client.add_resource(ssl_sock.sock)
response = http_call(ssl_sock, request_method, proxy.fetch_server, request_headers, payload)
client.add_resource(response.rfile)
client.add_resource(response.counted_sock)
response.app_status = response.status
response.app_options = response.getheader('X-GOA-Options', '')
if response.status != 200:
return response
data = response.read(4)
if len(data) < 4:
response.status = 502
response.fp = io.BytesIO(b'connection aborted. too short leadtype data=' + data)
response.read = response.fp.read
return response
response.status, headers_length = struct.unpack('!hh', data)
data = response.read(headers_length)
if len(data) < headers_length:
response.status = 502
response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)
response.read = response.fp.read
return response
if 'rc4' not in response.app_options:
response.msg = httplib.HTTPMessage(io.BytesIO(zlib.decompress(data, -zlib.MAX_WBITS)))
else:
response.msg = httplib.HTTPMessage(io.BytesIO(zlib.decompress(rc4crypt(data, password), -zlib.MAX_WBITS)))
if password and response.fp:
response.fp = RC4FileObject(response.fp, password)
return response
def gae_urlfetch_for_3_2_0_or_above(client, proxy, method, url, headers, body):
def inflate(data):
return zlib.decompress(data, -zlib.MAX_WBITS)
def deflate(data):
return zlib.compress(data)[2:-4]
password=proxy.password
is_obfuscate_enabled=proxy.is_obfuscate_enabled
is_rc4_enabled=proxy.is_rc4_enabled
if isinstance(body, basestring) and body:
if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
zbody = deflate(body)
if len(zbody) < len(body):
body = zbody
headers['Content-Encoding'] = 'deflate'
headers['Content-Length'] = str(len(body))
# GAE donot allow set `Host` header
if 'Host' in headers:
del headers['Host']
kwargs = {}
if password:
kwargs['password'] = password
payload = '%s %s %s\r\n' % (method, url, 'HTTP/1.0')
payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in SKIP_HEADERS)
payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v) for k, v in kwargs.items() if v)
# prepare GAE request
request_method = 'POST'
request_headers = {}
fetch_server = proxy.fetch_server
if is_obfuscate_enabled:
request_method = 'GET'
fetch_server += 'ps/%d%s.gif' % (int(time.time()*1000), random.random())
request_headers['X-URLFETCH-PS1'] = base64.b64encode(deflate(payload)).strip()
if body:
request_headers['X-URLFETCH-PS2'] = base64.b64encode(deflate(body)).strip()
body = ''
else:
payload = deflate(payload)
body = '%s%s%s' % (struct.pack('!h', len(payload)), payload, body)
if is_rc4_enabled:
request_headers['X-URLFETCH-Options'] = 'rc4'
body = rc4crypt(body, password)
request_headers['Content-Length'] = str(len(body))
# post data
ssl_sock = create_ssl_connection()
ssl_sock.counter = stat.opened(ssl_sock, proxy, host=client.host, ip=client.dst_ip)
LOGGER.info('[%s] urlfetch %s %s via %s %0.2f'
% (repr(client), method, url, ssl_sock.google_ip, get_google_ip_latency(ssl_sock.google_ip)))
client.add_resource(ssl_sock)
client.add_resource(ssl_sock.counter)
client.add_resource(ssl_sock.sock)
response = http_call(ssl_sock, request_method, fetch_server, request_headers, body)
client.add_resource(response.rfile)
client.add_resource(response.counted_sock)
response.app_status = response.status
if response.status != 200:
return response
if is_rc4_enabled:
response.fp = RC4FileObject(response.fp, password)
data = response.read(2)
if len(data) < 2:
response.status = 502
response.fp = io.BytesIO(b'connection aborted. too short leadbyte data=' + data)
response.read = response.fp.read
return response
headers_length, = struct.unpack('!h', data)
data = response.read(headers_length)
if len(data) < headers_length:
response.status = 502
response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)
response.read = response.fp.read
return response
raw_response_line, headers_data = inflate(data).split('\r\n', 1)
_, response.status, response.reason = raw_response_line.split(None, 2)
response.status = int(response.status)
response.reason = response.reason.strip()
response.msg = httplib.HTTPMessage(io.BytesIO(headers_data))
return response