def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
method = kwargs['method']
url = kwargs['url']
#logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get('password', ''):
start_response('403 Forbidden', [('Content-type', 'text/html')])
return [gae_error_html(errno='403', error='Wrong password.', description='GoAgent proxy.ini password is wrong!')]
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
start_response('501 Unsupported', [('Content-type', 'text/html')])
return [gae_error_html(errno='501', error=('Invalid Method: '+str(method)), description='Unsupported Method')]
deadline = Deadline
headers = dict(headers)
headers['Connection'] = 'close'
payload = environ['wsgi.input'].read() if 'Content-Length' in headers else None
errors = []
for i in xrange(int(kwargs.get('fetchmax', FetchMax))):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%s)', deadline, url, response and response.headers)
if response and response.headers.get('content-length'):
response.status_code = 206
response.headers['accept-ranges'] = 'bytes'
response.headers['content-range'] = 'bytes 0-%d/%s' % (len(response.content)-1, response.headers['content-length'])
response.headers['content-length'] = len(response.content)
break
else:
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i==0 and method=='GET':
deadline = Deadline * 2
else:
start_response('500 Internal Server Error', [('Content-type', 'text/html')])
return [gae_error_html(errno='502', error=('Python Urlfetch Error: ' + str(method)), description=str(errors))]
if 'content-encoding' not in response.headers and response.headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
compressobj = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
response_headers = [('Set-Cookie', encode_request(response.headers, status=str(response.status_code), encoding='gzip'))]
start_response('200 OK', response_headers)
return ['\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff', compressobj.compress(response.content), compressobj.flush(), struct.pack('<LL', zlib.crc32(response.content)&0xFFFFFFFFL, len(response.content)&0xFFFFFFFFL)]
else:
if 'content-length' not in response.headers:
response_headers = [('Set-Cookie', encode_request(response.headers, status=str(response.status_code), length=str(len(response.content))))]
else:
response_headers = [('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))]
start_response('200 OK', response_headers)
return [response.content]
def gae_post(environ, start_response):
data = zlib.decompress(environ['wsgi.input'].read(
int(environ['CONTENT_LENGTH'])))
request = dict((k, binascii.a2b_hex(v))
for k, _, v in (x.partition('=') for x in data.split('&')))
#logging.debug('post() get fetch request %s', request)
method = request['method']
url = request['url']
payload = request['payload']
if __password__ and __password__ != request.get('password', ''):
return send_notify(start_response, method, url, 403, 'Wrong password.')
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
return send_notify(start_response, method, url, 501, 'Invalid Method')
deadline = Deadline
headers = dict(
(k.title(), v.lstrip())
for k, _, v in (line.partition(':')
for line in request['headers'].splitlines()))
headers['Connection'] = 'close'
errors = []
for i in xrange(FetchMax if 'fetchmax' not in
request else int(request['fetchmax'])):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers,
False, False, deadline, False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' %
(e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)',
deadline, url)
time.sleep(1)
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
except urlfetch.InvalidURLError as e:
return send_notify(start_response, method, url, 501,
'Invalid URL: %s' % e)
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error(
'ResponseTooLargeError(deadline=%s, url=%r) response(%r)',
deadline, url, response)
m = re.search(r'=\s*(\d+)-',
headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start,
start + FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append('Exception %s(deadline=%s)' % (e, deadline))
else:
return send_notify(start_response, method, url, 500,
'Python Server: Urlfetch error: %s' % errors)
headers = response.headers
if 'set-cookie' in headers:
scs = headers['set-cookie'].split(', ')
cookies = []
i = -1
for sc in scs:
if re.match(r'[^ =]+ ', sc):
try:
cookies[i] = '%s, %s' % (cookies[i], sc)
except IndexError:
pass
else:
cookies.append(sc)
i += 1
headers['set-cookie'] = '\r\nSet-Cookie: '.join(cookies)
if 'content-length' not in headers:
headers['content-length'] = str(len(response.content))
headers['connection'] = 'close'
return send_response(start_response, response.status_code, headers,
response.content)
def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
method = kwargs['method']
url = kwargs['url']
#logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get('password', ''):
start_response('403 Forbidden', [('Content-Type', 'text/html')])
return [gae_error_html(errno='403', error='Wrong password.', description='GoAgent proxy.ini password is wrong!')]
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
start_response('501 Unsupported', [('Content-Type', 'text/html')])
return [gae_error_html(errno='501', error=('Invalid Method: '+str(method)), description='Unsupported Method')]
deadline = Deadline
headers = dict(headers)
headers['Connection'] = 'close'
payload = environ['wsgi.input'].read() if 'Content-Length' in headers else None
accept_encoding = headers.get('Accept-Encoding', '')
errors = []
for i in xrange(int(kwargs.get('fetchmax', FetchMax))):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%s)', deadline, url, response and response.headers)
if response and response.headers.get('content-length'):
response.status_code = 206
response.headers['accept-ranges'] = 'bytes'
response.headers['content-range'] = 'bytes 0-%d/%s' % (len(response.content)-1, response.headers['content-length'])
response.headers['content-length'] = len(response.content)
break
else:
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i==0 and method=='GET':
deadline = Deadline * 2
else:
start_response('500 Internal Server Error', [('Content-Type', 'text/html')])
return [gae_error_html(errno='502', error=('Python Urlfetch Error: ' + str(method)), description=str(errors))]
#logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, response.status_code, dict(response.headers), response.content[:1024])
if 'content-encoding' not in response.headers and len(response.content) < DeflateMaxSize and 'deflate' in accept_encoding and response.headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
zdata = zlib.compress(response.content)[2:-4]
response.headers['Content-Length'] = str(len(zdata))
response.headers['Content-Encoding'] = 'deflate'
start_response('200 OK', [('Content-Type', 'image/gif'), ('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))])
return [zdata]
else:
response.headers['Content-Length'] = str(len(response.content))
start_response('200 OK', [('Content-Type', 'image/gif'), ('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))])
return [response.content]
def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ["HTTP_COOKIE"])
method = kwargs["method"]
url = kwargs["url"]
# logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get("password", ""):
start_response("403 Forbidden", [("Content-type", "text/html")])
return [
gae_error_html(errno="403", error="Wrong password.", description="GoAgent proxy.ini password is wrong!")
]
fetchmethod = getattr(urlfetch, method, "")
if not fetchmethod:
start_response("501 Unsupported", [("Content-type", "text/html")])
return [gae_error_html(errno="501", error=("Invalid Method: " + str(method)), description="Unsupported Method")]
deadline = Deadline
headers = dict(headers)
headers["Connection"] = "close"
payload = environ["wsgi.input"].read() if "Content-Length" in headers else None
errors = []
for i in xrange(int(kwargs.get("fetchmax", FetchMax))):
try:
response = urlfetch.fetch(
url,
payload,
fetchmethod,
headers,
allow_truncated=False,
follow_redirects=False,
deadline=deadline,
validate_certificate=False,
)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append("DeadlineExceededError %s(deadline=%s)" % (e, deadline))
logging.error("DeadlineExceededError(deadline=%s, url=%r)", deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append("DownloadError %s(deadline=%s)" % (e, deadline))
logging.error("DownloadError(deadline=%s, url=%r)", deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error(
"ResponseTooLargeError(deadline=%s, url=%r) response(%s)", deadline, url, response and response.headers
)
if response and response.headers.get("content-length"):
response.status_code = 206
response.headers["accept-ranges"] = "bytes"
response.headers["content-range"] = "bytes 0-%d/%s" % (
len(response.content) - 1,
response.headers["content-length"],
)
response.headers["content-length"] = len(response.content)
break
else:
m = re.search(r"=\s*(\d+)-", headers.get("Range") or headers.get("range") or "")
if m is None:
headers["Range"] = "bytes=0-%d" % FetchMaxSize
else:
headers.pop("Range", "")
headers.pop("range", "")
start = int(m.group(1))
headers["Range"] = "bytes=%s-%d" % (start, start + FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i == 0 and method == "GET":
deadline = Deadline * 2
else:
start_response("500 Internal Server Error", [("Content-type", "text/html")])
return [gae_error_html(errno="502", error=("Python Urlfetch Error: " + str(method)), description=str(errors))]
if "content-encoding" not in response.headers and response.headers.get("content-type", "").startswith(
("text/", "application/json", "application/javascript")
):
compressobj = zlib.compressobj(
zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0
)
response_headers = [
("Set-Cookie", encode_request(response.headers, status=str(response.status_code), encoding="gzip"))
]
start_response("200 OK", response_headers)
return [
"\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff",
compressobj.compress(response.content),
compressobj.flush(),
struct.pack("<LL", zlib.crc32(response.content) & 0xFFFFFFFFL, len(response.content) & 0xFFFFFFFFL),
]
else:
if "content-length" not in response.headers:
response_headers = [
(
"Set-Cookie",
encode_request(
response.headers, status=str(response.status_code), length=str(len(response.content))
),
)
]
else:
response_headers = [("Set-Cookie", encode_request(response.headers, status=str(response.status_code)))]
start_response("200 OK", response_headers)
return [response.content]
def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
method = kwargs['method']
url = kwargs['url']
#logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get('password', ''):
start_response('403 Forbidden', [('Content-Type', 'text/html')])
return [gae_error_html(errno='403', error='Wrong password.', description='GoAgent proxy.ini password is wrong!')]
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
start_response('501 Unsupported', [('Content-Type', 'text/html')])
return [gae_error_html(errno='501', error=('Invalid Method: '+str(method)), description='Unsupported Method')]
deadline = Deadline
headers = dict(headers)
headers['Connection'] = 'close'
payload = environ['wsgi.input'].read() if 'Content-Length' in headers else None
accept_encoding = headers.get('Accept-Encoding', '')
errors = []
for i in xrange(int(kwargs.get('fetchmax', FetchMax))):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%r)', deadline, url, response)
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % int(kwargs.get('fetchmaxsize', FetchMaxSize))
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+int(kwargs.get('fetchmaxsize', FetchMaxSize)))
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i==0 and method=='GET':
deadline = Deadline * 2
else:
start_response('500 Internal Server Error', [('Content-Type', 'text/html')])
return [gae_error_html(errno='502', error=('Python Urlfetch Error: ' + str(method)), description=str(errors))]
#logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, response.status_code, dict(response.headers), response.content[:1024])
data = response.content
if 'content-encoding' not in response.headers and len(response.content) < DeflateMaxSize and response.headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
if 'deflate' in accept_encoding:
response.headers['Content-Encoding'] = 'deflate'
data = zlib.compress(data)[2:-4]
elif 'gzip' in accept_encoding:
response.headers['Content-Encoding'] = 'gzip'
compressobj = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
dataio = cStringIO.StringIO()
dataio.write('\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff')
dataio.write(compressobj.compress(data))
dataio.write(compressobj.flush())
dataio.write(struct.pack('<LL', zlib.crc32(data)&0xFFFFFFFFL, len(data)&0xFFFFFFFFL))
data = dataio.getvalue()
response.headers['Content-Length'] = str(len(data))
start_response('200 OK', [('Content-Type', 'image/gif'), ('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))])
return [data]
def gae_post(environ, start_response):
data = zlib.decompress(environ['wsgi.input'].read(int(environ['CONTENT_LENGTH'])))
request = dict((k,binascii.a2b_hex(v)) for k, _, v in (x.partition('=') for x in data.split('&')))
#logging.debug('post() get fetch request %s', request)
method = request['method']
url = request['url']
payload = request['payload']
if __password__ and __password__ != request.get('password', ''):
return send_notify(start_response, method, url, 403, 'Wrong password.')
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
return send_notify(start_response, method, url, 501, 'Invalid Method')
deadline = Deadline
headers = dict((k.title(),v.lstrip()) for k, _, v in (line.partition(':') for line in request['headers'].splitlines()))
headers['Connection'] = 'close'
errors = []
for i in xrange(FetchMax if 'fetchmax' not in request else int(request['fetchmax'])):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, False, False, deadline, False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
except urlfetch.InvalidURLError as e:
return send_notify(start_response, method, url, 501, 'Invalid URL: %s' % e)
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%r)', deadline, url, response)
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append('Exception %s(deadline=%s)' % (e, deadline))
else:
return send_notify(start_response, method, url, 500, 'Python Server: Urlfetch error: %s' % errors)
headers = response.headers
if 'set-cookie' in headers:
scs = headers['set-cookie'].split(', ')
cookies = []
i = -1
for sc in scs:
if re.match(r'[^ =]+ ', sc):
try:
cookies[i] = '%s, %s' % (cookies[i], sc)
except IndexError:
pass
else:
cookies.append(sc)
i += 1
headers['set-cookie'] = '\r\nSet-Cookie: '.join(cookies)
if 'content-length' not in headers:
headers['content-length'] = str(len(response.content))
headers['connection'] = 'close'
return send_response(start_response, response.status_code, headers, response.content)
