def cors(self):
headers = self.request.headers
self.header('Access-Control-Allow-Origin', headers.get('Origin', '*'))
self.header('Access-Control-Allow-Credentials', 'true')
if self.method == 'OPTIONS' and 'Origin' in headers:
self._logger.debug('responding to CORS preflight request')
self.status(200)
self.header('Access-Control-Allow-Methods',
headers.get('Access-Control-Request-Method', ''))
self.header('Access-Control-Allow-Headers',
headers.get('Access-Control-Request-Headers', ''))
self.body('')
raise SkipRemainingRules()
else:
self.add_header('Vary', 'Origin')
def test_GET_home_HTML(self):
"Fetch the home page, in HTML format."
wr = self.get_wr('text/html')
response = wr.GET('/')
self.assertEqual(response.status, httplib.OK,
msg="HTTP status %s" % response.status)
headers = self.get_headers(response)
self.assert_(headers.get('content-type').startswith('text/html'))
def send_response(start_response, status, headers, content, content_type='image/gif'):
strheaders = '&'.join('%s=%s' % (k, binascii.b2a_hex(v)) for k, v in headers.iteritems() if v)
#logging.debug('response status=%s, headers=%s, content length=%d', status, headers, len(content))
if headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
data = '1' + zlib.compress('%s%s%s' % (struct.pack('>3I', status, len(strheaders), len(content)), strheaders, content))
else:
data = '0%s%s%s' % (struct.pack('>3I', status, len(strheaders), len(content)), strheaders, content)
start_response('200 OK', [('Content-type', content_type)])
return [data]
def header_value(environ, key):
headers = wsgiref.headers.Headers(environ)
values = headers.get(key)
if not values:
return ""
if isinstance(values, list): #pragma NO COVER can't be true under Py3k.
return ",".join(values)
else:
return values
def header_value(environ, key):
headers = wsgiref.headers.Headers(environ)
values = headers.get(key)
if not values:
return ""
if isinstance(values, list): #pragma NO COVER can't be true under Py3k.
return ",".join(values)
else:
return values
def send_response(start_response, status, headers, content, content_type='image/gif'):
headers['Content-Length'] = str(len(content))
strheaders = '&'.join('%s=%s' % (k, binascii.b2a_hex(v)) for k, v in headers.iteritems() if v)
#logging.debug('response status=%s, headers=%s, content length=%d', status, headers, len(content))
if headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
data = '1' + zlib.compress('%s%s%s' % (struct.pack('>3I', status, len(strheaders), len(content)), strheaders, content))
else:
data = '0%s%s%s' % (struct.pack('>3I', status, len(strheaders), len(content)), strheaders, content)
start_response('200 OK', [('Content-type', content_type)])
return [data]
def send_response(start_response, status, headers, content, content_type="image/gif"):
strheaders = "&".join("%s=%s" % (k, binascii.b2a_hex(v)) for k, v in headers.iteritems() if v)
# logging.debug('response status=%s, headers=%s, content length=%d', status, headers, len(content))
if headers.get("content-type", "").startswith(("text/", "application/json", "application/javascript")):
data = "1" + zlib.compress(
"%s%s%s" % (struct.pack(">3I", status, len(strheaders), len(content)), strheaders, content)
)
else:
data = "0%s%s%s" % (struct.pack(">3I", status, len(strheaders), len(content)), strheaders, content)
start_response("200 OK", [("Content-type", content_type)])
return [data]
def paas_application(environ, start_response):
try:
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
except Exception as e:
logging.exception(
"decode_request(environ['HTTP_COOKIE']=%r) failed: %s",
environ.get('HTTP_COOKIE'), e)
raise
if __password__ and __password__ != kwargs.get('password'):
url = 'https://goa%d%s' % (int(
time.time() * 100), environ['HTTP_HOST'])
response = httplib_request('GET', url, timeout=5)
status_line = '%s %s' % (response.status,
httplib.responses.get(response.status, 'OK'))
start_response(status_line, response.getheaders())
yield response.read()
raise StopIteration
method = kwargs['method']
url = kwargs['url']
timeout = Deadline
logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url,
'HTTP/1.1')
if method != 'CONNECT':
try:
headers = dict(headers)
headers['Connection'] = 'close'
data = environ['wsgi.input'] if int(
headers.get('Content-Length', 0)) else None
response = httplib_request(method,
url,
body=data,
headers=headers,
timeout=timeout)
response_headers = dict(response.getheaders())
response_headers['connection'] = 'close'
response_headers.pop('transfer-encoding', '')
start_response('%s OK' % response.status, response_headers.items())
bufsize = 8192
while 1:
data = response.read(bufsize)
if not data:
response.close()
break
yield data
except httplib.HTTPException as e:
raise
def paas_application(environ, start_response):
try:
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
except Exception as e:
logging.exception("decode_request(environ['HTTP_COOKIE']=%r) failed: %s", environ.get('HTTP_COOKIE'), e)
raise
if __password__ and __password__ != kwargs.get('password'):
url = 'https://goa%d%s' % (int(time.time()*100), environ['HTTP_HOST'])
response = httplib_request('GET', url, timeout=5)
status_line = '%s %s' % (response.status, httplib.responses.get(response.status, 'OK'))
start_response(status_line, response.getheaders())
yield response.read()
raise StopIteration
method = kwargs['method']
url = kwargs['url']
timeout = Deadline
logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if method != 'CONNECT':
try:
headers = dict(headers)
headers['Connection'] = 'close'
data = environ['wsgi.input'] if int(headers.get('Content-Length',0)) else None
response = httplib_request(method, url, body=data, headers=headers, timeout=timeout)
response_headers = dict(response.getheaders())
response_headers['connection'] = 'close'
response_headers.pop('transfer-encoding', '')
start_response('%s OK' % response.status, response_headers.items())
bufsize = 8192
while 1:
data = response.read(bufsize)
if not data:
response.close()
break
yield data
except httplib.HTTPException as e:
raise
def paas_application(environ, start_response):
try:
headers, kwargs = decode_request(environ["HTTP_COOKIE"])
except Exception as e:
logging.exception("decode_request(environ['HTTP_COOKIE']=%r) failed: %s", environ.get("HTTP_COOKIE"), e)
raise
if __password__ and __password__ != kwargs.get("password"):
url = "https://goa%d%s" % (int(time.time() * 100), environ["HTTP_HOST"])
response = httplib_request("GET", url, timeout=5)
status_line = "%s %s" % (response.status, httplib.responses.get(response.status, "OK"))
start_response(status_line, response.getheaders())
yield response.read()
raise StopIteration
method = kwargs["method"]
url = kwargs["url"]
timeout = Deadline
logging.info('%s "%s %s %s" - -', environ["REMOTE_ADDR"], method, url, "HTTP/1.1")
if method != "CONNECT":
try:
headers = dict(headers)
headers["Connection"] = "close"
data = environ["wsgi.input"] if int(headers.get("Content-Length", 0)) else None
response = httplib_request(method, url, body=data, headers=headers, timeout=timeout)
response_headers = dict(response.getheaders())
response_headers["connection"] = "close"
response_headers.pop("transfer-encoding", "")
start_response("%s OK" % response.status, response_headers.items())
bufsize = 8192
while 1:
data = response.read(bufsize)
if not data:
response.close()
break
yield data
except httplib.HTTPException as e:
raise
def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
method = kwargs['method']
url = kwargs['url']
#logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get('password', ''):
start_response('403 Forbidden', [('Content-type', 'text/html')])
return [gae_error_html(errno='403', error='Wrong password.', description='GoAgent proxy.ini password is wrong!')]
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
start_response('501 Unsupported', [('Content-type', 'text/html')])
return [gae_error_html(errno='501', error=('Invalid Method: '+str(method)), description='Unsupported Method')]
deadline = Deadline
headers = dict(headers)
headers['Connection'] = 'close'
payload = environ['wsgi.input'].read() if 'Content-Length' in headers else None
errors = []
for i in xrange(int(kwargs.get('fetchmax', FetchMax))):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%s)', deadline, url, response and response.headers)
if response and response.headers.get('content-length'):
response.status_code = 206
response.headers['accept-ranges'] = 'bytes'
response.headers['content-range'] = 'bytes 0-%d/%s' % (len(response.content)-1, response.headers['content-length'])
response.headers['content-length'] = len(response.content)
break
else:
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i==0 and method=='GET':
deadline = Deadline * 2
else:
start_response('500 Internal Server Error', [('Content-type', 'text/html')])
return [gae_error_html(errno='502', error=('Python Urlfetch Error: ' + str(method)), description=str(errors))]
if 'content-encoding' not in response.headers and response.headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
compressobj = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
response_headers = [('Set-Cookie', encode_request(response.headers, status=str(response.status_code), encoding='gzip'))]
start_response('200 OK', response_headers)
return ['\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff', compressobj.compress(response.content), compressobj.flush(), struct.pack('<LL', zlib.crc32(response.content)&0xFFFFFFFFL, len(response.content)&0xFFFFFFFFL)]
else:
if 'content-length' not in response.headers:
response_headers = [('Set-Cookie', encode_request(response.headers, status=str(response.status_code), length=str(len(response.content))))]
else:
response_headers = [('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))]
start_response('200 OK', response_headers)
return [response.content]
def gae_post(environ, start_response):
data = zlib.decompress(environ['wsgi.input'].read(
int(environ['CONTENT_LENGTH'])))
request = dict((k, binascii.a2b_hex(v))
for k, _, v in (x.partition('=') for x in data.split('&')))
#logging.debug('post() get fetch request %s', request)
method = request['method']
url = request['url']
payload = request['payload']
if __password__ and __password__ != request.get('password', ''):
return send_notify(start_response, method, url, 403, 'Wrong password.')
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
return send_notify(start_response, method, url, 501, 'Invalid Method')
deadline = Deadline
headers = dict(
(k.title(), v.lstrip())
for k, _, v in (line.partition(':')
for line in request['headers'].splitlines()))
headers['Connection'] = 'close'
errors = []
for i in xrange(FetchMax if 'fetchmax' not in
request else int(request['fetchmax'])):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers,
False, False, deadline, False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' %
(e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)',
deadline, url)
time.sleep(1)
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
except urlfetch.InvalidURLError as e:
return send_notify(start_response, method, url, 501,
'Invalid URL: %s' % e)
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error(
'ResponseTooLargeError(deadline=%s, url=%r) response(%r)',
deadline, url, response)
m = re.search(r'=\s*(\d+)-',
headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start,
start + FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append('Exception %s(deadline=%s)' % (e, deadline))
else:
return send_notify(start_response, method, url, 500,
'Python Server: Urlfetch error: %s' % errors)
headers = response.headers
if 'set-cookie' in headers:
scs = headers['set-cookie'].split(', ')
cookies = []
i = -1
for sc in scs:
if re.match(r'[^ =]+ ', sc):
try:
cookies[i] = '%s, %s' % (cookies[i], sc)
except IndexError:
pass
else:
cookies.append(sc)
i += 1
headers['set-cookie'] = '\r\nSet-Cookie: '.join(cookies)
if 'content-length' not in headers:
headers['content-length'] = str(len(response.content))
headers['connection'] = 'close'
return send_response(start_response, response.status_code, headers,
response.content)
def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
method = kwargs['method']
url = kwargs['url']
#logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get('password', ''):
start_response('403 Forbidden', [('Content-Type', 'text/html')])
return [gae_error_html(errno='403', error='Wrong password.', description='GoAgent proxy.ini password is wrong!')]
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
start_response('501 Unsupported', [('Content-Type', 'text/html')])
return [gae_error_html(errno='501', error=('Invalid Method: '+str(method)), description='Unsupported Method')]
deadline = Deadline
headers = dict(headers)
headers['Connection'] = 'close'
payload = environ['wsgi.input'].read() if 'Content-Length' in headers else None
accept_encoding = headers.get('Accept-Encoding', '')
errors = []
for i in xrange(int(kwargs.get('fetchmax', FetchMax))):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%s)', deadline, url, response and response.headers)
if response and response.headers.get('content-length'):
response.status_code = 206
response.headers['accept-ranges'] = 'bytes'
response.headers['content-range'] = 'bytes 0-%d/%s' % (len(response.content)-1, response.headers['content-length'])
response.headers['content-length'] = len(response.content)
break
else:
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i==0 and method=='GET':
deadline = Deadline * 2
else:
start_response('500 Internal Server Error', [('Content-Type', 'text/html')])
return [gae_error_html(errno='502', error=('Python Urlfetch Error: ' + str(method)), description=str(errors))]
#logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, response.status_code, dict(response.headers), response.content[:1024])
if 'content-encoding' not in response.headers and len(response.content) < DeflateMaxSize and 'deflate' in accept_encoding and response.headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
zdata = zlib.compress(response.content)[2:-4]
response.headers['Content-Length'] = str(len(zdata))
response.headers['Content-Encoding'] = 'deflate'
start_response('200 OK', [('Content-Type', 'image/gif'), ('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))])
return [zdata]
else:
response.headers['Content-Length'] = str(len(response.content))
start_response('200 OK', [('Content-Type', 'image/gif'), ('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))])
return [response.content]
def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ["HTTP_COOKIE"])
method = kwargs["method"]
url = kwargs["url"]
# logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get("password", ""):
start_response("403 Forbidden", [("Content-type", "text/html")])
return [
gae_error_html(errno="403", error="Wrong password.", description="GoAgent proxy.ini password is wrong!")
]
fetchmethod = getattr(urlfetch, method, "")
if not fetchmethod:
start_response("501 Unsupported", [("Content-type", "text/html")])
return [gae_error_html(errno="501", error=("Invalid Method: " + str(method)), description="Unsupported Method")]
deadline = Deadline
headers = dict(headers)
headers["Connection"] = "close"
payload = environ["wsgi.input"].read() if "Content-Length" in headers else None
errors = []
for i in xrange(int(kwargs.get("fetchmax", FetchMax))):
try:
response = urlfetch.fetch(
url,
payload,
fetchmethod,
headers,
allow_truncated=False,
follow_redirects=False,
deadline=deadline,
validate_certificate=False,
)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append("DeadlineExceededError %s(deadline=%s)" % (e, deadline))
logging.error("DeadlineExceededError(deadline=%s, url=%r)", deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append("DownloadError %s(deadline=%s)" % (e, deadline))
logging.error("DownloadError(deadline=%s, url=%r)", deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error(
"ResponseTooLargeError(deadline=%s, url=%r) response(%s)", deadline, url, response and response.headers
)
if response and response.headers.get("content-length"):
response.status_code = 206
response.headers["accept-ranges"] = "bytes"
response.headers["content-range"] = "bytes 0-%d/%s" % (
len(response.content) - 1,
response.headers["content-length"],
)
response.headers["content-length"] = len(response.content)
break
else:
m = re.search(r"=\s*(\d+)-", headers.get("Range") or headers.get("range") or "")
if m is None:
headers["Range"] = "bytes=0-%d" % FetchMaxSize
else:
headers.pop("Range", "")
headers.pop("range", "")
start = int(m.group(1))
headers["Range"] = "bytes=%s-%d" % (start, start + FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i == 0 and method == "GET":
deadline = Deadline * 2
else:
start_response("500 Internal Server Error", [("Content-type", "text/html")])
return [gae_error_html(errno="502", error=("Python Urlfetch Error: " + str(method)), description=str(errors))]
if "content-encoding" not in response.headers and response.headers.get("content-type", "").startswith(
("text/", "application/json", "application/javascript")
):
compressobj = zlib.compressobj(
zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0
)
response_headers = [
("Set-Cookie", encode_request(response.headers, status=str(response.status_code), encoding="gzip"))
]
start_response("200 OK", response_headers)
return [
"\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff",
compressobj.compress(response.content),
compressobj.flush(),
struct.pack("<LL", zlib.crc32(response.content) & 0xFFFFFFFFL, len(response.content) & 0xFFFFFFFFL),
]
else:
if "content-length" not in response.headers:
response_headers = [
(
"Set-Cookie",
encode_request(
response.headers, status=str(response.status_code), length=str(len(response.content))
),
)
]
else:
response_headers = [("Set-Cookie", encode_request(response.headers, status=str(response.status_code)))]
start_response("200 OK", response_headers)
return [response.content]
def gae_post_ex(environ, start_response):
headers, kwargs = decode_request(environ['HTTP_COOKIE'])
method = kwargs['method']
url = kwargs['url']
#logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1')
if __password__ and __password__ != kwargs.get('password', ''):
start_response('403 Forbidden', [('Content-Type', 'text/html')])
return [gae_error_html(errno='403', error='Wrong password.', description='GoAgent proxy.ini password is wrong!')]
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
start_response('501 Unsupported', [('Content-Type', 'text/html')])
return [gae_error_html(errno='501', error=('Invalid Method: '+str(method)), description='Unsupported Method')]
deadline = Deadline
headers = dict(headers)
headers['Connection'] = 'close'
payload = environ['wsgi.input'].read() if 'Content-Length' in headers else None
accept_encoding = headers.get('Accept-Encoding', '')
errors = []
for i in xrange(int(kwargs.get('fetchmax', FetchMax))):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
deadline = Deadline * 2
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%r)', deadline, url, response)
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % int(kwargs.get('fetchmaxsize', FetchMaxSize))
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+int(kwargs.get('fetchmaxsize', FetchMaxSize)))
deadline = Deadline * 2
except Exception as e:
errors.append(str(e))
if i==0 and method=='GET':
deadline = Deadline * 2
else:
start_response('500 Internal Server Error', [('Content-Type', 'text/html')])
return [gae_error_html(errno='502', error=('Python Urlfetch Error: ' + str(method)), description=str(errors))]
#logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, response.status_code, dict(response.headers), response.content[:1024])
data = response.content
if 'content-encoding' not in response.headers and len(response.content) < DeflateMaxSize and response.headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')):
if 'deflate' in accept_encoding:
response.headers['Content-Encoding'] = 'deflate'
data = zlib.compress(data)[2:-4]
elif 'gzip' in accept_encoding:
response.headers['Content-Encoding'] = 'gzip'
compressobj = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
dataio = cStringIO.StringIO()
dataio.write('\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff')
dataio.write(compressobj.compress(data))
dataio.write(compressobj.flush())
dataio.write(struct.pack('<LL', zlib.crc32(data)&0xFFFFFFFFL, len(data)&0xFFFFFFFFL))
data = dataio.getvalue()
response.headers['Content-Length'] = str(len(data))
start_response('200 OK', [('Content-Type', 'image/gif'), ('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))])
return [data]
def gae_post(environ, start_response):
data = zlib.decompress(environ['wsgi.input'].read(int(environ['CONTENT_LENGTH'])))
request = dict((k,binascii.a2b_hex(v)) for k, _, v in (x.partition('=') for x in data.split('&')))
#logging.debug('post() get fetch request %s', request)
method = request['method']
url = request['url']
payload = request['payload']
if __password__ and __password__ != request.get('password', ''):
return send_notify(start_response, method, url, 403, 'Wrong password.')
fetchmethod = getattr(urlfetch, method, '')
if not fetchmethod:
return send_notify(start_response, method, url, 501, 'Invalid Method')
deadline = Deadline
headers = dict((k.title(),v.lstrip()) for k, _, v in (line.partition(':') for line in request['headers'].splitlines()))
headers['Connection'] = 'close'
errors = []
for i in xrange(FetchMax if 'fetchmax' not in request else int(request['fetchmax'])):
try:
response = urlfetch.fetch(url, payload, fetchmethod, headers, False, False, deadline, False)
break
except apiproxy_errors.OverQuotaError as e:
time.sleep(4)
except urlfetch.DeadlineExceededError as e:
errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline))
logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
except urlfetch.DownloadError as e:
errors.append('DownloadError %s(deadline=%s)' % (e, deadline))
logging.error('DownloadError(deadline=%s, url=%r)', deadline, url)
time.sleep(1)
except urlfetch.InvalidURLError as e:
return send_notify(start_response, method, url, 501, 'Invalid URL: %s' % e)
except urlfetch.ResponseTooLargeError as e:
response = e.response
logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%r)', deadline, url, response)
m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '')
if m is None:
headers['Range'] = 'bytes=0-%d' % FetchMaxSize
else:
headers.pop('Range', '')
headers.pop('range', '')
start = int(m.group(1))
headers['Range'] = 'bytes=%s-%d' % (start, start+FetchMaxSize)
deadline = Deadline * 2
except Exception as e:
errors.append('Exception %s(deadline=%s)' % (e, deadline))
else:
return send_notify(start_response, method, url, 500, 'Python Server: Urlfetch error: %s' % errors)
headers = response.headers
if 'set-cookie' in headers:
scs = headers['set-cookie'].split(', ')
cookies = []
i = -1
for sc in scs:
if re.match(r'[^ =]+ ', sc):
try:
cookies[i] = '%s, %s' % (cookies[i], sc)
except IndexError:
pass
else:
cookies.append(sc)
i += 1
headers['set-cookie'] = '\r\nSet-Cookie: '.join(cookies)
if 'content-length' not in headers:
headers['content-length'] = str(len(response.content))
headers['connection'] = 'close'
return send_response(start_response, response.status_code, headers, response.content)
