def test_gzip_decode_limit(self): max_gzip_decode = 20 * 1024 * 1024 data = "\0" * max_gzip_decode encoded = xmlrpclib.gzip_encode(data) decoded = xmlrpclib.gzip_decode(encoded) self.assertEqual(len(decoded), max_gzip_decode) data = "\0" * (max_gzip_decode + 1) encoded = xmlrpclib.gzip_encode(data) with self.assertRaisesRegexp(ValueError, "max gzipped payload length exceeded"): xmlrpclib.gzip_decode(encoded) xmlrpclib.gzip_decode(encoded, max_decode=-1)
def test_gzip_decode_limit(self): max_gzip_decode = 20 * 1024 * 1024 data = '\0' * max_gzip_decode encoded = xmlrpclib.gzip_encode(data) decoded = xmlrpclib.gzip_decode(encoded) self.assertEqual(len(decoded), max_gzip_decode) data = '\0' * (max_gzip_decode + 1) encoded = xmlrpclib.gzip_encode(data) with self.assertRaisesRegexp(ValueError, "max gzipped payload length exceeded"): xmlrpclib.gzip_decode(encoded) xmlrpclib.gzip_decode(encoded, max_decode=-1)
def download_subtitle(self, filename, sub_id, ext): out_file=OpenSubtitles._sub_file(filename, self._lang, ext) res =self._proxy.DownloadSubtitles(self._token,[sub_id,]) self._parse_status(res) data=res.get('data') t=data[0]['data'] text=xmlrpclib.gzip_decode(base64.b64decode(t)) with open(out_file, 'wb') as f: f.write(text) return out_file
def download_subtitle(self, filename, sub_id, ext): out_file = OpenSubtitles._sub_file(filename, self._lang, ext) res = self._proxy.DownloadSubtitles(self._token, [ sub_id, ]) self._parse_status(res) data = res.get('data') t = data[0]['data'] text = xmlrpclib.gzip_decode(base64.b64decode(t)) with open(out_file, 'wb') as f: f.write(text) return out_file
def decode_request_content(self, data): encoding = self.headers.get('content-encoding', 'identity').lower() if encoding == 'identity': return data if encoding == 'gzip': try: return xmlrpclib.gzip_decode(data) except NotImplementedError: self.send_response(501, 'encoding %r not supported' % encoding) except ValueError: self.send_response(400, 'error decoding gzip content') else: self.send_response(501, 'encoding %r not supported' % encoding) self.send_header('Content-length', '0') self.end_headers()
def decode_request_content(self, data): #support gzip encoding of request encoding = self.headers.get("content-encoding", "identity").lower() if encoding == "identity": return data if encoding == "gzip": try: return xmlrpclib.gzip_decode(data) except NotImplementedError: self.send_response(501, "encoding %r not supported" % encoding) except ValueError: self.send_response(400, "error decoding gzip content") else: self.send_response(501, "encoding %r not supported" % encoding) self.send_header("Content-length", "0") self.end_headers()
def decode_request_content(self, data,environ,start_response): #support gzip encoding of request encoding = environ.get("HTTP_CONTENT_ENCODING", "identity").lower() if encoding == "identity": return data if encoding == "gzip": try: return xmlrpclib.gzip_decode(data) except NotImplementedError: response = "encoding %r not supported" % encoding start_response("%d %s" % (501,_RESPONSE_STATUSES[501]),[ ("Content-length", str(len(response))) ]) except ValueError: response = "error decoding gzip content" start_response("%d %s" % (400,_RESPONSE_STATUSES[400]),[ ("Content-length", str(len(response))) ]) else: response = "encoding %r not supported" % encoding start_response("%d %s" % (501,_RESPONSE_STATUSES[501]),[ ("Content-length", str(len(response))) ]) return [response]
def _run(self, defer): sock = socket.create_connection(self._address, 2) self._sock = sock defer(sock.close) def w(data): with self._w_lck: #log('%s[%s] %s' % (data[:3].decode(), id(sock), id(self._sock)), 'SEND') sock.sendall(data) bot_name = self._botname bot_id = 'bot.%s' % urn5(bot_name) w(('CONNECT {"name":"%s","verbose":false,"pedantic":false}\r\nSUB %s.* 2\r\n' % (bot_name, self.cli_id)).encode('utf8')) #r = sock.makefile('rb', 0) #defer(r.close) self._online.set() #print('_run', flush=True) #print(dir(sock)) self._fg_serve_forever = True c = 0 #cc = 0 while self._fg_serve_forever: #cc += 1 cmd = '' data = '' try: data = recvline(sock) cmd, data = data[:3], data[3:] #log(cmd, '_RUN') except socket.timeout: #log('timeout', '_RUN') c += 1 #log('%s) timeout' % c, 'socket0') if c > 2: c = 0 #log('pong) timeout', 'socket0') w(b'PONG\r\n') continue if not cmd: raise RuntimeError('[ Socket ] cmd is empty') if not data: raise RuntimeError('[ Socket ] data is empty') #if cc > 15: # raise RuntimeError('[ DEBUG ] error for test') #log('>%s<' % data, '<%s>' % cmd) if 'MSG' == cmd: #MSG <subject> <sid> [reply-to] <#bytes>\r\n[payload]\r\n data = data.split() #print('data:', data) if 3 == len(data): subj, sid, reply_id, size = data[0], data[1], '', int( data[2]) else: subj, sid, reply_id, size = data[0], data[1], data[2], int( data[3]) payload = recvall(sock, size) if size > 0 else b'' sock.recv(1) sock.recv(1) #print(cmd, subj, sid, reply_id, repr(payload)[:48], '...', len(payload)) if sid[:2] == '1.': sid = '2' if sid == '2': fg_http = b'HTTP' == payload[:4] if fg_http: pass else: if b'\x1f\x8b\x08\x00' == payload[:4]: if PY2: payload = gzip_decode(payload) else: payload = gzip_decode(payload, -1) if payload: try: payload = json.loads(payload, object_hook=_binary) except Exception as e: payload = {'error': str(e)} else: pyaload = {} i = int(subj.rsplit('.', 1)[-1], 16) _r = self._sync.get(i) #log(_r, i) if _r: #print('[ASK]:', _r.ask) #sys.stdout.flush() if not reply_id and size == 0: #print('1>', _r.is_set(), _r.ask) #sys.stdout.flush() _r.ask = True _r.set() else: #print('2>', _r.is_set(), _r.ask) #sys.stdout.flush() while _r.is_set() and _r.ask: time.sleep(0.001) if fg_http: headers, payload = payload.split(b'\r\n', 1) status, headers = json.loads(headers[4:]) #if b'\x1f\x8b\x08\x00' == payload[:4]: # payload = gzip_decode(payload, -1) _r.result = (status, headers, payload) _r.error = None else: _r.result = payload.pop('result', None) _r.error = payload.pop('error', None) _r.set() #else: # print(i, payload) else: if b'\x1f\x8b\x08\x00' == payload[:4]: if PY2: payload = gzip_decode(payload) else: payload = gzip_decode(payload, -1) if payload: payload = json.loads(payload, object_hook=_binary) #payload = pickle.loads(payload) else: payload = None #print(subj) i = int(sid.rsplit('.', 1)[-1], 16) _r = self._sync.get(i) #print('_r:', _r, i) #print(', '.join(str(x) for x in self._sync.keys())) if _r: _r.put((subj, payload)) #else: # print(i, payload) elif 'PIN' == cmd: w(b'PONG\r\n') elif 'INF' == cmd: self._info = json.loads(data[2:])
def _handle_client(self, reply_id, payload): #threading.current_thread().conn = client_c _fg = True while _fg: fg_http = False _fg = False try: if b'HTTP' == payload[:4]: head, payload = payload.split(b'\r\n', 1) head = json.loads(head[4:]) #if b'\x1f\x8b\x08\x00' == payload[:4]: # payload = gzip_decode(payload, -1) #print(head) #print(payload) func_name = '.http' args = (head, payload) kwargs = {} fg_http = True else: if b'\x1f\x8b\x08\x00' == payload[:4]: if PY2: payload = gzip_decode(payload) else: payload = gzip_decode(payload, -1) payload = json.loads(payload, object_hook=_binary) func_name = payload.get('method', '') args = payload.pop('args', []) kwargs = payload.pop('kwargs', {}) #except EOFError: # #print('close:', client_c) # #sys.stdout.flush() # break except Exception as e: print('recv:', type(e), str(e)) traceback.print_exc() sys.stdout.flush() break #print(111) try: func = None if func_name in self._functions: func = self._functions[func_name] else: for fn in func_name.split('.'): func = getattr(func, fn) if func else self._functions[fn] #print(222, func) if func: if callable(func): r = func(*args, **kwargs) """ if isinstance(r, types.GeneratorType): self._send(reply_id, {'result': list}) # types.ListType) #client_c.send('types.GeneratorType') for v in r: self._send(reply_id, {'result': v}) self._send(reply_id, {'result': StopIteration}) continue """ else: r = func if fg_http: _len = self._send(reply_id, r, fg_http=True) else: _len = self._send(reply_id, {'result': r}) else: r = RuntimeError('%s not found' % func_name) _len = self._send(reply_id, {'error': str(r)}) #print('send >>', _len) except Exception as e: try: self._send(reply_id, {'error': str(e)}) except IOError: break except Exception as e: print('send:', type(e), str(e)) sys.stdout.flush() break
def _http(self, head, body): global __appname__,__profile__, __version__, __index__ status = '200 OK' r = b'' uri, args = head[''][1] method = uri.rsplit('/', 1)[-1] if '/' == uri: try: appname = getattr(sys, '__appname__') except AttributeError: appname = self._botname.split('.', 1)[0] #appname = __appname__ try: profile = getattr(sys, '__profile__') except AttributeError: profile = self._botname.split('.', 1)[-1] #profile = __profile__ try: version = getattr(sys, '__version__') except AttributeError: version = __version__ try: _index = getattr(sys, '__index__') except AttributeError: _index = __index__ headers = [("Content-Type", "text/plain; charset=utf-8"), ("Cache-Control", "no-cache"), ("Access-Control-Allow-Origin", "*")] r = ('%s %s %s.%s %s %s:%s/%s' % (time.strftime(_ts), sys.__hostname__, appname,profile, version, _index,self.request_number,self.request_count)).encode('utf-8') headers.append(('Content-Length', str(len(r)))) return status, headers, r if uri == '/ui': uri = 'ui/' return '303 OK', [('Location', uri),], b'' if uri[:4] == '/ui/' and uri[:8] != '/ui/RPC2': return '404 Not Found', [], b'404 Not Found' headers = [('Content-Type', 'application/json; charset=utf-8'), ("Cache-Control", "no-cache"), ("Access-Control-Allow-Origin", "*")] try: if 'post' == head[''][0]: if b'\x1f\x8b\x08\x00' == body[:4]: if PY2: body = gzip_decode(body) else: body = gzip_decode(body, -1) body = json.loads(body) method, _a, _kw = body['method'], body.get('params', []), body.get('kwargs', {}) else: _a, _kw = [], {} if args: #for a in args.split('&'): # a = a.split('=', 1) for a in (a.split('=', 1) for arg in args.split('&') for a in arg.split(',')): if len(a) > 1: _kw[a[0]] = a[1] else: _a.append(a[0]) r = self._func(method, _a, _kw) r = json.dumps({'result': r}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8') except Exception as e: log(None) r = json.dumps({'error': str(e)}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8') #r = json.dumps({'error': traceback.format_exc()}, ensure_ascii=False, cls=ExtJSONEncoder).encode('utf8') finally: if len(r) > 1400: r = gzip_encode(r) headers.append(('Content-Encoding', 'gzip')) headers.append(('Content-Length', str(len(r)))) return status, headers, r