def test_gzip_decode_limit(self):
max_gzip_decode = 20 * 1024 * 1024
data = "\0" * max_gzip_decode
encoded = xmlrpclib.gzip_encode(data)
decoded = xmlrpclib.gzip_decode(encoded)
self.assertEqual(len(decoded), max_gzip_decode)
data = "\0" * (max_gzip_decode + 1)
encoded = xmlrpclib.gzip_encode(data)
with self.assertRaisesRegexp(ValueError, "max gzipped payload length exceeded"):
xmlrpclib.gzip_decode(encoded)
xmlrpclib.gzip_decode(encoded, max_decode=-1)
def test_gzip_decode_limit(self):
max_gzip_decode = 20 * 1024 * 1024
data = '\0' * max_gzip_decode
encoded = xmlrpclib.gzip_encode(data)
decoded = xmlrpclib.gzip_decode(encoded)
self.assertEqual(len(decoded), max_gzip_decode)
data = '\0' * (max_gzip_decode + 1)
encoded = xmlrpclib.gzip_encode(data)
with self.assertRaisesRegexp(ValueError,
"max gzipped payload length exceeded"):
xmlrpclib.gzip_decode(encoded)
xmlrpclib.gzip_decode(encoded, max_decode=-1)
def download_subtitle(self, filename, sub_id, ext):
out_file=OpenSubtitles._sub_file(filename, self._lang, ext)
res =self._proxy.DownloadSubtitles(self._token,[sub_id,])
self._parse_status(res)
data=res.get('data')
t=data[0]['data']
text=xmlrpclib.gzip_decode(base64.b64decode(t))
with open(out_file, 'wb') as f:
f.write(text)
return out_file
def download_subtitle(self, filename, sub_id, ext):
out_file = OpenSubtitles._sub_file(filename, self._lang, ext)
res = self._proxy.DownloadSubtitles(self._token, [
sub_id,
])
self._parse_status(res)
data = res.get('data')
t = data[0]['data']
text = xmlrpclib.gzip_decode(base64.b64decode(t))
with open(out_file, 'wb') as f:
f.write(text)
return out_file
def decode_request_content(self, data):
encoding = self.headers.get('content-encoding', 'identity').lower()
if encoding == 'identity':
return data
if encoding == 'gzip':
try:
return xmlrpclib.gzip_decode(data)
except NotImplementedError:
self.send_response(501, 'encoding %r not supported' % encoding)
except ValueError:
self.send_response(400, 'error decoding gzip content')
else:
self.send_response(501, 'encoding %r not supported' % encoding)
self.send_header('Content-length', '0')
self.end_headers()
def decode_request_content(self, data):
#support gzip encoding of request
encoding = self.headers.get("content-encoding", "identity").lower()
if encoding == "identity":
return data
if encoding == "gzip":
try:
return xmlrpclib.gzip_decode(data)
except NotImplementedError:
self.send_response(501, "encoding %r not supported" % encoding)
except ValueError:
self.send_response(400, "error decoding gzip content")
else:
self.send_response(501, "encoding %r not supported" % encoding)
self.send_header("Content-length", "0")
self.end_headers()
def decode_request_content(self, data):
#support gzip encoding of request
encoding = self.headers.get("content-encoding", "identity").lower()
if encoding == "identity":
return data
if encoding == "gzip":
try:
return xmlrpclib.gzip_decode(data)
except NotImplementedError:
self.send_response(501, "encoding %r not supported" % encoding)
except ValueError:
self.send_response(400, "error decoding gzip content")
else:
self.send_response(501, "encoding %r not supported" % encoding)
self.send_header("Content-length", "0")
self.end_headers()
def decode_request_content(self, data):
encoding = self.headers.get('content-encoding', 'identity').lower()
if encoding == 'identity':
return data
if encoding == 'gzip':
try:
return xmlrpclib.gzip_decode(data)
except NotImplementedError:
self.send_response(501, 'encoding %r not supported' % encoding)
except ValueError:
self.send_response(400, 'error decoding gzip content')
else:
self.send_response(501, 'encoding %r not supported' % encoding)
self.send_header('Content-length', '0')
self.end_headers()
def decode_request_content(self, data,environ,start_response):
#support gzip encoding of request
encoding = environ.get("HTTP_CONTENT_ENCODING", "identity").lower()
if encoding == "identity":
return data
if encoding == "gzip":
try:
return xmlrpclib.gzip_decode(data)
except NotImplementedError:
response = "encoding %r not supported" % encoding
start_response("%d %s" % (501,_RESPONSE_STATUSES[501]),[
("Content-length", str(len(response)))
])
except ValueError:
response = "error decoding gzip content"
start_response("%d %s" % (400,_RESPONSE_STATUSES[400]),[
("Content-length", str(len(response)))
])
else:
response = "encoding %r not supported" % encoding
start_response("%d %s" % (501,_RESPONSE_STATUSES[501]),[
("Content-length", str(len(response)))
])
return [response]
def _run(self, defer):
sock = socket.create_connection(self._address, 2)
self._sock = sock
defer(sock.close)
def w(data):
with self._w_lck:
#log('%s[%s] %s' % (data[:3].decode(), id(sock), id(self._sock)), 'SEND')
sock.sendall(data)
bot_name = self._botname
bot_id = 'bot.%s' % urn5(bot_name)
w(('CONNECT {"name":"%s","verbose":false,"pedantic":false}\r\nSUB %s.* 2\r\n'
% (bot_name, self.cli_id)).encode('utf8'))
#r = sock.makefile('rb', 0)
#defer(r.close)
self._online.set()
#print('_run', flush=True)
#print(dir(sock))
self._fg_serve_forever = True
c = 0
#cc = 0
while self._fg_serve_forever:
#cc += 1
cmd = ''
data = ''
try:
data = recvline(sock)
cmd, data = data[:3], data[3:]
#log(cmd, '_RUN')
except socket.timeout:
#log('timeout', '_RUN')
c += 1
#log('%s) timeout' % c, 'socket0')
if c > 2:
c = 0
#log('pong) timeout', 'socket0')
w(b'PONG\r\n')
continue
if not cmd:
raise RuntimeError('[ Socket ] cmd is empty')
if not data:
raise RuntimeError('[ Socket ] data is empty')
#if cc > 15:
# raise RuntimeError('[ DEBUG ] error for test')
#log('>%s<' % data, '<%s>' % cmd)
if 'MSG' == cmd:
#MSG <subject> <sid> [reply-to] <#bytes>\r\n[payload]\r\n
data = data.split()
#print('data:', data)
if 3 == len(data):
subj, sid, reply_id, size = data[0], data[1], '', int(
data[2])
else:
subj, sid, reply_id, size = data[0], data[1], data[2], int(
data[3])
payload = recvall(sock, size) if size > 0 else b''
sock.recv(1)
sock.recv(1)
#print(cmd, subj, sid, reply_id, repr(payload)[:48], '...', len(payload))
if sid[:2] == '1.':
sid = '2'
if sid == '2':
fg_http = b'HTTP' == payload[:4]
if fg_http:
pass
else:
if b'\x1f\x8b\x08\x00' == payload[:4]:
if PY2:
payload = gzip_decode(payload)
else:
payload = gzip_decode(payload, -1)
if payload:
try:
payload = json.loads(payload,
object_hook=_binary)
except Exception as e:
payload = {'error': str(e)}
else:
pyaload = {}
i = int(subj.rsplit('.', 1)[-1], 16)
_r = self._sync.get(i)
#log(_r, i)
if _r:
#print('[ASK]:', _r.ask)
#sys.stdout.flush()
if not reply_id and size == 0:
#print('1>', _r.is_set(), _r.ask)
#sys.stdout.flush()
_r.ask = True
_r.set()
else:
#print('2>', _r.is_set(), _r.ask)
#sys.stdout.flush()
while _r.is_set() and _r.ask:
time.sleep(0.001)
if fg_http:
headers, payload = payload.split(b'\r\n', 1)
status, headers = json.loads(headers[4:])
#if b'\x1f\x8b\x08\x00' == payload[:4]:
# payload = gzip_decode(payload, -1)
_r.result = (status, headers, payload)
_r.error = None
else:
_r.result = payload.pop('result', None)
_r.error = payload.pop('error', None)
_r.set()
#else:
# print(i, payload)
else:
if b'\x1f\x8b\x08\x00' == payload[:4]:
if PY2:
payload = gzip_decode(payload)
else:
payload = gzip_decode(payload, -1)
if payload:
payload = json.loads(payload, object_hook=_binary)
#payload = pickle.loads(payload)
else:
payload = None
#print(subj)
i = int(sid.rsplit('.', 1)[-1], 16)
_r = self._sync.get(i)
#print('_r:', _r, i)
#print(', '.join(str(x) for x in self._sync.keys()))
if _r:
_r.put((subj, payload))
#else:
# print(i, payload)
elif 'PIN' == cmd:
w(b'PONG\r\n')
elif 'INF' == cmd:
self._info = json.loads(data[2:])
def _handle_client(self, reply_id, payload):
#threading.current_thread().conn = client_c
_fg = True
while _fg:
fg_http = False
_fg = False
try:
if b'HTTP' == payload[:4]:
head, payload = payload.split(b'\r\n', 1)
head = json.loads(head[4:])
#if b'\x1f\x8b\x08\x00' == payload[:4]:
# payload = gzip_decode(payload, -1)
#print(head)
#print(payload)
func_name = '.http'
args = (head, payload)
kwargs = {}
fg_http = True
else:
if b'\x1f\x8b\x08\x00' == payload[:4]:
if PY2:
payload = gzip_decode(payload)
else:
payload = gzip_decode(payload, -1)
payload = json.loads(payload, object_hook=_binary)
func_name = payload.get('method', '')
args = payload.pop('args', [])
kwargs = payload.pop('kwargs', {})
#except EOFError:
# #print('close:', client_c)
# #sys.stdout.flush()
# break
except Exception as e:
print('recv:', type(e), str(e))
traceback.print_exc()
sys.stdout.flush()
break
#print(111)
try:
func = None
if func_name in self._functions:
func = self._functions[func_name]
else:
for fn in func_name.split('.'):
func = getattr(func, fn) if func else self._functions[fn]
#print(222, func)
if func:
if callable(func):
r = func(*args, **kwargs)
"""
if isinstance(r, types.GeneratorType):
self._send(reply_id, {'result': list}) # types.ListType)
#client_c.send('types.GeneratorType')
for v in r:
self._send(reply_id, {'result': v})
self._send(reply_id, {'result': StopIteration})
continue
"""
else:
r = func
if fg_http:
_len = self._send(reply_id, r, fg_http=True)
else:
_len = self._send(reply_id, {'result': r})
else:
r = RuntimeError('%s not found' % func_name)
_len = self._send(reply_id, {'error': str(r)})
#print('send >>', _len)
except Exception as e:
try:
self._send(reply_id, {'error': str(e)})
except IOError:
break
except Exception as e:
print('send:', type(e), str(e))
sys.stdout.flush()
break
def _http(self, head, body):
global __appname__,__profile__, __version__, __index__
status = '200 OK'
r = b''
uri, args = head[''][1]
method = uri.rsplit('/', 1)[-1]
if '/' == uri:
try:
appname = getattr(sys, '__appname__')
except AttributeError:
appname = self._botname.split('.', 1)[0]
#appname = __appname__
try:
profile = getattr(sys, '__profile__')
except AttributeError:
profile = self._botname.split('.', 1)[-1]
#profile = __profile__
try:
version = getattr(sys, '__version__')
except AttributeError:
version = __version__
try:
_index = getattr(sys, '__index__')
except AttributeError:
_index = __index__
headers = [("Content-Type", "text/plain; charset=utf-8"), ("Cache-Control", "no-cache"), ("Access-Control-Allow-Origin", "*")]
r = ('%s %s %s.%s %s %s:%s/%s' % (time.strftime(_ts), sys.__hostname__, appname,profile, version, _index,self.request_number,self.request_count)).encode('utf-8')
headers.append(('Content-Length', str(len(r))))
return status, headers, r
if uri == '/ui':
uri = 'ui/'
return '303 OK', [('Location', uri),], b''
if uri[:4] == '/ui/' and uri[:8] != '/ui/RPC2':
return '404 Not Found', [], b'404 Not Found'
headers = [('Content-Type', 'application/json; charset=utf-8'), ("Cache-Control", "no-cache"), ("Access-Control-Allow-Origin", "*")]
try:
if 'post' == head[''][0]:
if b'\x1f\x8b\x08\x00' == body[:4]:
if PY2:
body = gzip_decode(body)
else:
body = gzip_decode(body, -1)
body = json.loads(body)
method, _a, _kw = body['method'], body.get('params', []), body.get('kwargs', {})
else:
_a, _kw = [], {}
if args:
#for a in args.split('&'):
# a = a.split('=', 1)
for a in (a.split('=', 1) for arg in args.split('&') for a in arg.split(',')):
if len(a) > 1:
_kw[a[0]] = a[1]
else:
_a.append(a[0])
r = self._func(method, _a, _kw)
r = json.dumps({'result': r}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
except Exception as e:
log(None)
r = json.dumps({'error': str(e)}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
#r = json.dumps({'error': traceback.format_exc()}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8')
finally:
if len(r) > 1400:
r = gzip_encode(r)
headers.append(('Content-Encoding', 'gzip'))
headers.append(('Content-Length', str(len(r))))
return status, headers, r
