def test_gzip_decode_limit(self):
max_gzip_decode = 20 * 1024 * 1024
data = "\0" * max_gzip_decode
encoded = xmlrpclib.gzip_encode(data)
decoded = xmlrpclib.gzip_decode(encoded)
self.assertEqual(len(decoded), max_gzip_decode)
data = "\0" * (max_gzip_decode + 1)
encoded = xmlrpclib.gzip_encode(data)
with self.assertRaisesRegexp(ValueError, "max gzipped payload length exceeded"):
xmlrpclib.gzip_decode(encoded)
xmlrpclib.gzip_decode(encoded, max_decode=-1)
def test_gzip_decode_limit(self):
max_gzip_decode = 20 * 1024 * 1024
data = '\0' * max_gzip_decode
encoded = xmlrpclib.gzip_encode(data)
decoded = xmlrpclib.gzip_decode(encoded)
self.assertEqual(len(decoded), max_gzip_decode)
data = '\0' * (max_gzip_decode + 1)
encoded = xmlrpclib.gzip_encode(data)
with self.assertRaisesRegexp(ValueError,
"max gzipped payload length exceeded"):
xmlrpclib.gzip_decode(encoded)
xmlrpclib.gzip_decode(encoded, max_decode=-1)
def do_POST(self):
"""Handles the HTTP POST request.
Attempts to interpret all HTTP POST requests as XML-RPC calls,
which are forwarded to the server's _dispatch method for handling.
"""
if not self.is_rpc_path_valid():
self.report_404()
return
else:
try:
max_chunk_size = 10485760
size_remaining = int(self.headers['content-length'])
L = []
while size_remaining:
chunk_size = min(size_remaining, max_chunk_size)
chunk = self.rfile.read(chunk_size)
if not chunk:
break
L.append(chunk)
size_remaining -= len(L[-1])
data = ''.join(L)
data = self.decode_request_content(data)
if data is None:
return
response = self.server._marshaled_dispatch(
data, getattr(self, '_dispatch', None), self.path)
except Exception as e:
self.send_response(500)
if hasattr(self.server, '_send_traceback_header'
) and self.server._send_traceback_header:
self.send_header('X-exception', str(e))
self.send_header('X-traceback', traceback.format_exc())
self.send_header('Content-length', '0')
self.end_headers()
else:
self.send_response(200)
self.send_header('Content-type', 'text/xml')
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get('gzip', 0)
if q:
try:
response = xmlrpclib.gzip_encode(response)
self.send_header('Content-Encoding', 'gzip')
except NotImplementedError:
pass
self.send_header('Content-length', str(len(response)))
self.end_headers()
self.wfile.write(response)
return
def do_POST(self):
"""Handles the HTTP POST request.
Attempts to interpret all HTTP POST requests as XML-RPC calls,
which are forwarded to the server's _dispatch method for handling.
"""
if not self.is_rpc_path_valid():
self.report_404()
return
else:
try:
max_chunk_size = 10485760
size_remaining = int(self.headers['content-length'])
L = []
while size_remaining:
chunk_size = min(size_remaining, max_chunk_size)
chunk = self.rfile.read(chunk_size)
if not chunk:
break
L.append(chunk)
size_remaining -= len(L[-1])
data = ''.join(L)
data = self.decode_request_content(data)
if data is None:
return
response = self.server._marshaled_dispatch(data, getattr(self, '_dispatch', None), self.path)
except Exception as e:
self.send_response(500)
if hasattr(self.server, '_send_traceback_header') and self.server._send_traceback_header:
self.send_header('X-exception', str(e))
self.send_header('X-traceback', traceback.format_exc())
self.send_header('Content-length', '0')
self.end_headers()
else:
self.send_response(200)
self.send_header('Content-type', 'text/xml')
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get('gzip', 0)
if q:
try:
response = xmlrpclib.gzip_encode(response)
self.send_header('Content-Encoding', 'gzip')
except NotImplementedError:
pass
self.send_header('Content-length', str(len(response)))
self.end_headers()
self.wfile.write(response)
return
def _send(self, inbox_id, obj, fg_http=False):
if fg_http:
data = b''.join([b'HTTP', json.dumps(obj[:2], ensure_ascii=False, separators=(',', ':')).encode('utf8'), b'\r\n', obj[2]])
else:
data = json.dumps(obj, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
if len(data) > 1400:
data = gzip_encode(data)
data = b'PUB %s %s\r\n%s\r\n' % (inbox_id.encode(), ('%s' % len(data)).encode(), data)
with self._w_lck:
#log(repr(data), 'send2')
self._sock.sendall(data)
return len(data)
def single_request(self, host, handler, request_body, verbose=0):
proto = ('http', 'https')[bool(self.https)]
if request_body:
request_body=xmlrpclib.gzip_encode(request_body)
req = urllib2.Request('%s://%s%s' % (proto, host, handler), request_body)
req.add_header('User-agent', self.user_agent)
req.add_header("Accept-Encoding", "gzip")
if request_body:
req.add_header("Content-Encoding", 'gzip')
self.verbose = verbose
resp=self.opener.open(req)
if resp.headers.get("Content-Encoding") == 'gzip':
resp = gzip.GzipFile(fileobj=StringIO(resp.read()), mode='rb')
return self.parse_response(resp)
def _return_html(self, response):
self.send_response(200)
self.send_header("Content-type", "text/html")
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get("gzip", 0)
if q:
try:
response = xmlrpclib.gzip_encode(response)
self.send_header("Content-Encoding", "gzip")
except NotImplementedError:
pass
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def single_request(self, host, handler, request_body, verbose=0):
proto = ('http', 'https')[bool(self.https)]
if request_body:
request_body = xmlrpclib.gzip_encode(request_body)
req = urllib2.Request('%s://%s%s' % (proto, host, handler),
request_body)
req.add_header('User-agent', self.user_agent)
req.add_header("Accept-Encoding", "gzip")
if request_body:
req.add_header("Content-Encoding", 'gzip')
self.verbose = verbose
resp = self.opener.open(req)
if resp.headers.get("Content-Encoding") == 'gzip':
resp = gzip.GzipFile(fileobj=StringIO(resp.read()), mode='rb')
return self.parse_response(resp)
def notify(self, subject, data=None):
if not self._fg_serve_forever:
return
if data is None:
data = ('PUB %s 0\r\n\r\n' % subject).encode('utf8')
else:
data = json.dumps(data, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
#data = pickle.dumps(data, protocol=2)
if len(data) > 1400:
data = gzip_encode(data)
data = ('PUB %s %s\r\n' % (subject, len(data))).encode('utf8') + data + b'\r\n'
with self._w_lck:
try:
self._sock.sendall(data)
return True
except:
pass
def _return_html(self, response):
gzipped = False
self.send_response(200)
self.send_header("Content-type", "text/html")
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get("gzip", 0)
if q:
try:
response = gzip_encode(response.encode('utf-8'))
gzipped = True
self.send_header("Content-Encoding", "gzip")
except NotImplementedError:
pass
self.send_header("Content-length", str(len(response)))
self.end_headers()
if gzipped:
# It is already a byte buffer
self.wfile.write(response)
else:
self.wfile.write(response.encode('utf-8'))
def do_POST(self):
'''处理HTTP的POST请求
'''
if not self.is_rpc_path_valid():
self.report_404()
return
try:
max_chunk_size = 10 * 1024 * 1024
size_remaining = int(self.headers["content-length"])
L = []
while size_remaining:
chunk_size = min(size_remaining, max_chunk_size)
L.append(self.rfile.read(chunk_size))
size_remaining -= len(L[-1])
data = ''.join(L)
response = self.server._marshaled_dispatch(
data, getattr(self, '_dispatch', None), self.path)
self.send_response(200)
except Exception:
response = Fault().response()
self.send_response(500, response)
logger.get_logger().exception("ProtocolError:%s" % response)
if response is None:
response = ''
self.send_header("Content-Type", "application/json-rpc")
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get("gzip", 0)
if q:
try:
response = xmlrpclib.gzip_encode(response)
self.send_header("Content-Encoding", "gzip")
except NotImplementedError:
pass
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
self.server._send_traceback_header:
self.send_header("X-exception", str(e))
self.send_header("X-traceback", traceback.format_exc())
self.send_header("Content-length", "0")
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get("gzip", 0)
if q:
try:
response = xmlrpclib.gzip_encode(response)
self.send_header("Content-Encoding", "gzip")
except NotImplementedError:
pass
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def decode_request_content(self, data):
#support gzip encoding of request
encoding = self.headers.get("content-encoding", "identity").lower()
if encoding == "identity":
return data
if encoding == "gzip":
try:
return xmlrpclib.gzip_decode(data)
self.server._send_traceback_header:
self.send_header("X-exception", str(e))
self.send_header("X-traceback", traceback.format_exc())
self.send_header("Content-length", "0")
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get("gzip", 0)
if q:
try:
response = xmlrpclib.gzip_encode(response)
self.send_header("Content-Encoding", "gzip")
except NotImplementedError:
pass
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def decode_request_content(self, data):
#support gzip encoding of request
encoding = self.headers.get("content-encoding", "identity").lower()
if encoding == "identity":
return data
if encoding == "gzip":
try:
return xmlrpclib.gzip_decode(data)
def send(self, name, args, kwargs, sync=True, bot_id=None, botname=None):
#print('send:', name, args, kwargs, sync, bot_id)
if not bot_id:
bot_id = self.bot_id
if '.http' == name:
#if len(args[1]) > 1400 and b'\x1f\x8b\x08\x00' != args[1][:4]:
# data = b''.join([b'HTTP', json.dumps(args[0], ensure_ascii=False, separators=(',', ':')).encode('utf8'), b'\r\n', gzip_encode(args[1])])
#else:
data = b''.join([
b'HTTP',
json.dumps(args[0], ensure_ascii=False,
separators=(',', ':')).encode('utf8'), b'\r\n',
args[1]
])
else:
data = {'method': name, 'args': args, 'kwargs': kwargs}
data = json.dumps(data, ensure_ascii=False,
cls=ExtJSONEncoder).encode('utf8')
if len(data) > 1400:
data = gzip_encode(data)
if not self._online.wait(10):
raise RuntimeError('[ %s ] connect timeout' % str(self._address))
"""
c = 0
while not self._fg_serve_forever:
c += 1
if c > 10000:
raise RuntimeError('[ %s ] connect timeout' % str(self._address))
time.sleep(0.001)
"""
r = Event()
r.ask = False
r.result = None
r.error = None
i = id(r)
#cli, sid = 'cli.%s.%x' % (uuid.uuid4().hex, i), '1.%x' % i
#cmd = ('SUB %(cli)s %(sid)s\r\nUNSUB %(sid)s 2\r\nPUB %(bot)s %(cli)s %(l)s\r\n%%s\r\n' % {'cli': cli, 'sid': sid, 'l': len(data), 'bot': bot_id}).encode()
#print(cmd)
#data = cmd % data
data = ('PUB %s %s.%x %s\r\n' % (bot_id, self.cli_id, i, len(data))
).encode('utf8') + data + b'\r\n'
self._sync[i] = r
err = None
with self._w_lck:
#self._send_count += 1
try:
#if 0 == self._send_count % 10:
# self._sock.close()
# raise ValueError('self._send_count: %s' % self._send_count)
self._sock.sendall(data)
except Exception as e:
err = e
#self.cli_id = 'cli.%s' % uuid.uuid4().hex
self._sock.close()
self._online.clear()
log(None)
if err:
log('[ %s ] reconnect' % str(self._address))
if not self._online.wait(5):
raise RuntimeError('[ %s ] connect timeout' %
str(self._address))
with self._w_lck:
self._sock.sendall(data)
#print('self._send_count:', self._send_count, flush=True)
if type(sync) in (list, tuple):
_wait1, _wait2 = sync[:2]
else:
_wait1, _wait2 = 5, 40
if r.wait(_wait1):
r.clear()
else:
r.error = ServiceError('Service <%s> not found' %
(botname if botname else self._botname))
#with self._w_lck:
# self._sock.sendall(('UNSUB %s\r\n' % sid).encode())
if sync:
del self._sync[i]
raise r.error
if sync:
if not _wait2:
return r
if r.wait(_wait2) != True:
r.error = MethodError('Method <%s> timeout' % name)
#with self._w_lck:
# self._sock.sendall(('UNSUB %s\r\n' % sid).encode())
del self._sync[i]
if r.error:
if isinstance(r.error, (str, unicode)):
raise RuntimeError(r.error)
raise r.error
return r.result
else:
return r
def do_POST(self):
"""Handles the HTTP POST request.
Attempts to interpret all HTTP POST requests as XML-RPC calls,
which are forwarded to the server's _dispatch method for handling.
"""
# Check that the path is legal
if not self.is_rpc_path_valid():
self.report_404()
return
try:
# Get arguments by reading body of request.
# We read this in chunks to avoid straining
# socket.read(); around the 10 or 15Mb mark, some platforms
# begin to have problems (bug #792570).
max_chunk_size = 10 * 1024 * 1024
size_remaining = int(self.headers["content-length"])
L = []
while size_remaining:
chunk_size = min(size_remaining, max_chunk_size)
chunk = self.rfile.read(chunk_size)
if not chunk:
break
L.append(chunk)
size_remaining -= len(L[-1])
data = ''.join(L)
data = self.decode_request_content(data)
if data is None:
return #response has been sent
# In previous versions of SimpleXMLRPCServer, _dispatch
# could be overridden in this class, instead of in
# SimpleXMLRPCDispatcher. To maintain backwards compatibility,
# check to see if a subclass implements _dispatch and dispatch
# using that method if present.
response = self.server._marshaled_dispatch(
data, getattr(self, '_dispatch', None), self.path)
except Exception as e: # This should only happen if the module is buggy
# internal error, report as HTTP server error
self.send_response(500)
# Send information about the exception if requested
if hasattr(self.server, '_send_traceback_header') and \
self.server._send_traceback_header:
self.send_header("X-exception", str(e))
self.send_header("X-traceback", traceback.format_exc())
self.send_header("Content-length", "0")
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get("gzip", 0)
if q:
try:
response = xmlrpclib.gzip_encode(response)
self.send_header("Content-Encoding", "gzip")
except NotImplementedError:
pass
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def _http(self, head, body):
global __appname__,__profile__, __version__, __index__
status = '200 OK'
r = b''
uri, args = head[''][1]
method = uri.rsplit('/', 1)[-1]
if '/' == uri:
try:
appname = getattr(sys, '__appname__')
except AttributeError:
appname = self._botname.split('.', 1)[0]
#appname = __appname__
try:
profile = getattr(sys, '__profile__')
except AttributeError:
profile = self._botname.split('.', 1)[-1]
#profile = __profile__
try:
version = getattr(sys, '__version__')
except AttributeError:
version = __version__
try:
_index = getattr(sys, '__index__')
except AttributeError:
_index = __index__
headers = [("Content-Type", "text/plain; charset=utf-8"), ("Cache-Control", "no-cache"), ("Access-Control-Allow-Origin", "*")]
r = ('%s %s %s.%s %s %s:%s/%s' % (time.strftime(_ts), sys.__hostname__, appname,profile, version, _index,self.request_number,self.request_count)).encode('utf-8')
headers.append(('Content-Length', str(len(r))))
return status, headers, r
if uri == '/ui':
uri = 'ui/'
return '303 OK', [('Location', uri),], b''
if uri[:4] == '/ui/' and uri[:8] != '/ui/RPC2':
return '404 Not Found', [], b'404 Not Found'
headers = [('Content-Type', 'application/json; charset=utf-8'), ("Cache-Control", "no-cache"), ("Access-Control-Allow-Origin", "*")]
try:
if 'post' == head[''][0]:
if b'\x1f\x8b\x08\x00' == body[:4]:
if PY2:
body = gzip_decode(body)
else:
body = gzip_decode(body, -1)
body = json.loads(body)
method, _a, _kw = body['method'], body.get('params', []), body.get('kwargs', {})
else:
_a, _kw = [], {}
if args:
#for a in args.split('&'):
# a = a.split('=', 1)
for a in (a.split('=', 1) for arg in args.split('&') for a in arg.split(',')):
if len(a) > 1:
_kw[a[0]] = a[1]
else:
_a.append(a[0])
r = self._func(method, _a, _kw)
r = json.dumps({'result': r}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
except Exception as e:
log(None)
r = json.dumps({'error': str(e)}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
#r = json.dumps({'error': traceback.format_exc()}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
finally:
if len(r) > 1400:
r = gzip_encode(r)
headers.append(('Content-Encoding', 'gzip'))
headers.append(('Content-Length', str(len(r))))
return status, headers, r