def testDecompressStream(self): data = os.urandom(16 * 1024) compressed = zlib.compress(data) fp = StringIO.StringIO(compressed) dfo = util.decompressStream(fp) check = dfo.read() self.assertEqual(check, data) fp = StringIO.StringIO(compressed) dfo = util.decompressStream(fp) chunk = dfo.read(333) self.assertEqual(chunk, data[:333]) # test readline data = 'hello world\nhello world line 2\n' compressed = zlib.compress(data) fp = StringIO.StringIO(compressed) dfo = util.decompressStream(fp) line = dfo.readline() self.assertEqual(line, 'hello world\n') line = dfo.readline() self.assertEqual(line, 'hello world line 2\n') fp = StringIO.StringIO(compressed) dfo = util.decompressStream(fp) line = dfo.readline(5) self.assertEqual(line, 'hello') line = dfo.readline(5) self.assertEqual(line, ' worl') line = dfo.readline() self.assertEqual(line, 'd\n')
def _unwrapRPC(request): stream = request.body_file encoding = request.headers.get('Content-Encoding', 'identity') if encoding == 'deflate': stream = util.decompressStream(stream) stream.seek(0) elif encoding != 'identity': raise exc.HTTPBadRequest("Wrong Content-Encoding for XMLRPC request") return stream
def _handleResponse(self, req, response): fp = response encoding = response.getheader('content-encoding', None) if encoding == 'deflate': fp = util.decompressStream(fp) fp.seek(0) elif encoding == 'gzip': fp = util.GzipFile(fileobj=fp) fp.seek(0) return ResponseWrapper(fp, response)
def _unwrapRPC(request): stream = request.body_file encoding = request.headers.get('Content-Encoding', 'identity') if encoding == 'deflate': stream = util.decompressStream(stream) stream.seek(0) elif encoding != 'identity': raise exc.HTTPBadRequest( "Wrong Content-Encoding for XMLRPC request") return stream
def _handleResponse(self, req, response): fp = response encoding = response.getheader('content-encoding', None) if encoding == 'deflate': # disable until performace is better #fp = DecompressFileObj(fp) fp = util.decompressStream(fp) fp.seek(0) elif encoding == 'gzip': fp = util.GzipFile(fileobj=fp) fp.seek(0) return ResponseWrapper(fp, response)
def rpcHandler(context): req = context.req # only handle POSTs if req.method.upper() != 'POST': raise web_exc.HTTPMethodNotAllowed(allow='POST') if req.content_type == 'text/xml': kind = 'xml' elif req.content_type == 'application/x-json': kind = 'json' else: raise web_exc.HTTPBadRequest() # instantiate a MintServer srvr = server.MintServer(context.cfg, allowPrivate=True, req=req, db=context.db) stream = req.body_file encoding = req.headers.get('Content-Encoding', 'identity') if encoding == 'deflate': stream = util.decompressStream(stream) stream.seek(0) elif encoding != 'identity': raise web_exc.HTTPBadRequest() if kind == 'xml': (args, method) = util.xmlrpcLoad(stream) else: (method, args) = json.load(stream) # coax parameters into something MintServer likes params = [method, context.authToken, args] # go for it; return 403 if permission is denied try: # result is (isError, returnValues) result = srvr.callWrapper(*params) except (errors.InsufficientPermission, mint_error.PermissionDenied): return context.responseFactory( "<h1>Forbidden</h1>\n" "<p>Access denied by the server.</p>\n", status='403 Forbidden', content_type='text/html') # create a response if kind == 'xml': resp = xmlrpclib.dumps((result,), methodresponse=1) else: resp = json.dumps(result[1]) return context.responseFactory(content_type=req.content_type, body=resp)
def testCompressDecompressStream(self): # Test that compressing and uncompressing streams produces the same # data fd, tempf = tempfile.mkstemp() os.unlink(tempf) sio = os.fdopen(fd, "w+") # Some data we will compress for fn in ['distcache-1.4.5-2.src.rpm', 'distcc-2.9.tar.bz2', 'initscripts-10-11.src.rpm', 'jcd.iso']: util.copyStream(file(os.path.join(resources.get_archive(), fn)), sio) sio.seek(0) cstr = util.compressStream(sio) cstr.seek(0) dstr = util.decompressStream(cstr) dstr.seek(0) sio.seek(0) self.assertEqual(sio.read(), dstr.read())
def postRpc(self): if self.request.content_type != 'text/xml': return self._makeError('400 Bad Request', "Unrecognized Content-Type") stream = self.request.body_file encoding = self.request.headers.get('Content-Encoding', 'identity') if encoding == 'deflate': stream = util.decompressStream(stream) stream.seek(0) elif encoding != 'identity': return self._makeError('400 Bad Request', "Unrecognized Content-Encoding") try: params, method = util.xmlrpcLoad(stream) except: return self._makeError('400 Bad Request', "Malformed XMLRPC request") localAddr = '%s:%s' % (socket.gethostname(), self.getLocalPort()) try: request = self.requestFilter.fromWire(params) except (TypeError, ValueError, IndexError): return self._makeError('400 Bad Request', "Malformed XMLRPC arguments") rawUrl = self.request.url scheme = self.request.headers.get('X-Conary-Proxy-Target-Scheme') if scheme in ('http', 'https'): rawUrl = str(URL(rawUrl)._replace(scheme=scheme)) # Execution phase -- locate and call the target method try: responseArgs, extraInfo = self.proxyServer.callWrapper( protocol=None, port=None, methodname=method, authToken=self.auth, request=request, remoteIp=self.auth.remote_ip, rawUrl=rawUrl, localAddr=localAddr, protocolString=self.request.http_version, headers=self.request.headers, isSecure=self.isSecure) except errors.InsufficientPermission: return self._makeError('403 Forbidden', "Insufficient permission") rawResponse, headers = responseArgs.toWire(request.version) if extraInfo: headers['Via'] = proxy.formatViaHeader(localAddr, self.request.http_version, prefix=extraInfo.getVia()) response = self.responseFactory(headerlist=headers.items()) response.content_type = 'text/xml' # Output phase -- serialize and write the response body = util.xmlrpcDump((rawResponse, ), methodresponse=1) accept = self.request.accept_encoding if len(body) > 200 and 'deflate' in accept: response.content_encoding = 'deflate' response.body = zlib.compress(body, 5) else: response.body = body if (method == 'getChangeSet' and request.version >= 71 and not responseArgs.isException and response.status_int == 200 and responseArgs.result[0] and 'multipart/mixed' in list(self.request.accept)): return self.inlineChangeset(response, responseArgs, headers) else: return response
def handleXml(self, authToken): contentLength = int(self.headers['Content-Length']) sio = util.BoundedStringIO() actual = util.copyStream(self.rfile, sio, contentLength) if contentLength != actual: raise Exception(contentLength, actual) sio.seek(0) encoding = self.headers.get('Content-Encoding', None) if encoding == 'deflate': sio = util.decompressStream(sio) sio.seek(0) (params, method) = util.xmlrpcLoad(sio) logMe(3, "decoded xml-rpc call %s from %d bytes request" %(method, contentLength)) if self.netProxy: repos = self.netProxy else: repos = self.netRepos localHost, localPort = self.request.getsockname()[:2] if ':' in localHost: localHost = '[%s]' % localHost localAddr = '%s:%s' % (localHost, localPort) request = xmlshims.RequestArgs.fromWire(params) if repos is not None: try: response, extraInfo = repos.callWrapper( protocol='http', port=None, methodname=method, authToken=authToken, request=request, remoteIp=self.connection.getpeername()[0], rawUrl=self.path, localAddr=localAddr, protocolString=self.request_version, headers=self.headers, isSecure=self.server.isSecure, ) except errors.InsufficientPermission: self.send_error(403) return None except: # exceptions are handled (logged) in callWrapper - send # 500 code back to the client to indicate an error happened self.send_error(500) from conary.lib import formattrace excType, excValue, excTb = sys.exc_info() formattrace.formatTrace(excType, excValue, excTb, withLocals = False) return None logMe(3, "returned from", method) rawResponse, headers = response.toWire(request.version) sio = util.BoundedStringIO() util.xmlrpcDump((rawResponse,), stream = sio, methodresponse=1) respLen = sio.tell() logMe(3, "encoded xml-rpc response to %d bytes" % respLen) self.send_response(200) encoding = self.headers.get('Accept-encoding', '') if respLen > 200 and 'deflate' in encoding: sio.seek(0) sio = util.compressStream(sio, level = 5) respLen = sio.tell() self.send_header('Content-encoding', 'deflate') self.send_header("Content-type", "text/xml") self.send_header("Content-length", str(respLen)) for key, value in sorted(headers.items()): self.send_header(key, value) if extraInfo: # If available, send to the client the via headers all the way up # to us self.send_header('Via', proxy.formatViaHeader(localAddr, 'HTTP/1.0', prefix=extraInfo.getVia())) self.end_headers() sio.seek(0) util.copyStream(sio, self.wfile) logMe(3, "sent response to client", respLen, "bytes") return respLen
def post(port, isSecure, repos, req, authToken=None, repServer=None): if authToken is None: authToken = getAuth(req) if authToken is None: return apache.HTTP_BAD_REQUEST if authToken[0] != "anonymous" and not isSecure and repos.cfg.forceSSL: return apache.HTTP_FORBIDDEN if isSecure: protocol = "https" else: protocol = "http" extraInfo = None repos.log.reset() if req.headers_in['Content-Type'] == "text/xml": # handle XML-RPC requests encoding = req.headers_in.get('Content-Encoding', None) sio = util.BoundedStringIO() try: util.copyStream(req, sio) except IOError, e: # if we got a read timeout, marshal an exception back # to the client print >> sys.stderr, 'error reading from client: %s' %e method = 'unknown - client timeout' response = xmlshims.ResponseArgs.newException('ClientTimeout', 'The server was not able to read the ' 'XML-RPC request sent by this client. ' 'This is sometimes caused by MTU problems ' 'on your network connection. Using a ' 'smaller MTU may work around this ' 'problem.') headers = {} startTime = time.time() else: # otherwise, we've read the data, let's process it if encoding == 'deflate': sio.seek(0) try: sio = util.decompressStream(sio) except zlib.error, error: req.log_error("zlib inflate error in POST: %s" % error) return apache.HTTP_BAD_REQUEST startTime = time.time() sio.seek(0) try: (params, method) = util.xmlrpcLoad(sio) except: req.log_error('error parsing XMLRPC request') return apache.HTTP_BAD_REQUEST repos.log(3, "decoding=%s" % method, authToken[0], "%.3f" % (time.time()-startTime)) # req.connection.local_addr[0] is the IP address the server # listens on, not the IP address of the accepted socket. Most of # the times it will be 0.0.0.0 which is not very useful. We're # using local_ip instead, and we grab just the port from # local_addr. localAddr = "%s:%s" % (req.connection.local_ip, req.connection.local_addr[1]) remoteIp = req.connection.remote_ip # Get the IP address of the original request in the case # of a proxy, otherwise use the connection's remote_ip if 'X-Forwarded-For' in req.headers_in: # pick the right-most client, since that is # the one closest to us. For example, if # we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7" # we want to use 4.5.6.7 clients = req.headers_in['X-Forwarded-For'] remoteIp = clients.split(',')[-1].strip() try: request = xmlshims.RequestArgs.fromWire(params) except (TypeError, ValueError, IndexError): req.log_error('error parsing XMLRPC arguments') return apache.HTTP_BAD_REQUEST try: response, extraInfo = repos.callWrapper( protocol=protocol, port=port, methodname=method, authToken=authToken, request=request, remoteIp=remoteIp, rawUrl=req.unparsed_uri, localAddr=localAddr, protocolString=req.protocol, headers=req.headers_in, isSecure=isSecure, ) except errors.InsufficientPermission: return apache.HTTP_FORBIDDEN
def postRpc(self): if self.request.content_type != 'text/xml': return self._makeError('400 Bad Request', "Unrecognized Content-Type") stream = self.request.body_file encoding = self.request.headers.get('Content-Encoding', 'identity') if encoding == 'deflate': stream = util.decompressStream(stream) stream.seek(0) elif encoding != 'identity': return self._makeError('400 Bad Request', "Unrecognized Content-Encoding") try: params, method = util.xmlrpcLoad(stream) except: return self._makeError('400 Bad Request', "Malformed XMLRPC request") localAddr = socket.gethostname() try: request = self.requestFilter.fromWire(params) except (TypeError, ValueError, IndexError): return self._makeError('400 Bad Request', "Malformed XMLRPC arguments") # Execution phase -- locate and call the target method try: response, extraInfo = self.proxyServer.callWrapper( protocol=None, port=None, methodname=method, authToken=self.auth, request=request, remoteIp=self.auth.remote_ip, rawUrl=self.request.url, localAddr=localAddr, protocolString=self.request.http_version, headers=self.request.headers, isSecure=self.isSecure) except errors.InsufficientPermission: return self._makeError('403 Forbidden', "Insufficient permission") rawResponse, headers = response.toWire(request.version) response = self.responseFactory( headerlist=headers.items(), content_type='text/xml', ) # Output phase -- serialize and write the response body = util.xmlrpcDump((rawResponse, ), methodresponse=1) accept = self.request.accept_encoding if len(body) > 200 and 'deflate' in accept: response.content_encoding = 'deflate' response.body = zlib.compress(body, 5) else: response.body = body if extraInfo: headers['Via'] = proxy.formatViaHeader(localAddr, self.request.http_version, prefix=extraInfo.getVia()) return response
def handleXml(self, authToken): contentLength = int(self.headers['Content-Length']) sio = util.BoundedStringIO() actual = util.copyStream(self.rfile, sio, contentLength) if contentLength != actual: raise Exception(contentLength, actual) sio.seek(0) encoding = self.headers.get('Content-Encoding', None) if encoding == 'deflate': sio = util.decompressStream(sio) sio.seek(0) (params, method) = util.xmlrpcLoad(sio) logMe( 3, "decoded xml-rpc call %s from %d bytes request" % (method, contentLength)) if self.netProxy: repos = self.netProxy else: repos = self.netRepos localHost, localPort = self.request.getsockname()[:2] if ':' in localHost: localHost = '[%s]' % localHost localAddr = '%s:%s' % (localHost, localPort) request = xmlshims.RequestArgs.fromWire(params) if repos is not None: try: response, extraInfo = repos.callWrapper( protocol='http', port=None, methodname=method, authToken=authToken, request=request, remoteIp=self.connection.getpeername()[0], rawUrl=self.path, localAddr=localAddr, protocolString=self.request_version, headers=self.headers, isSecure=self.server.isSecure, ) except errors.InsufficientPermission: self.send_error(403) return None except: # exceptions are handled (logged) in callWrapper - send # 500 code back to the client to indicate an error happened self.send_error(500) from conary.lib import formattrace excType, excValue, excTb = sys.exc_info() formattrace.formatTrace(excType, excValue, excTb, withLocals=False) return None logMe(3, "returned from", method) rawResponse, headers = response.toWire(request.version) sio = util.BoundedStringIO() util.xmlrpcDump((rawResponse, ), stream=sio, methodresponse=1) respLen = sio.tell() logMe(3, "encoded xml-rpc response to %d bytes" % respLen) self.send_response(200) encoding = self.headers.get('Accept-encoding', '') if respLen > 200 and 'deflate' in encoding: sio.seek(0) sio = util.compressStream(sio, level=5) respLen = sio.tell() self.send_header('Content-encoding', 'deflate') self.send_header("Content-type", "text/xml") self.send_header("Content-length", str(respLen)) for key, value in sorted(headers.items()): self.send_header(key, value) if extraInfo: # If available, send to the client the via headers all the way up # to us self.send_header( 'Via', proxy.formatViaHeader(localAddr, 'HTTP/1.0', prefix=extraInfo.getVia())) self.end_headers() sio.seek(0) util.copyStream(sio, self.wfile) logMe(3, "sent response to client", respLen, "bytes") return respLen
def postRpc(self): if self.request.content_type != 'text/xml': return self._makeError('400 Bad Request', "Unrecognized Content-Type") stream = self.request.body_file encoding = self.request.headers.get('Content-Encoding', 'identity') if encoding == 'deflate': stream = util.decompressStream(stream) stream.seek(0) elif encoding != 'identity': return self._makeError('400 Bad Request', "Unrecognized Content-Encoding") try: params, method = util.xmlrpcLoad(stream) except: return self._makeError('400 Bad Request', "Malformed XMLRPC request") localAddr = '%s:%s' % (socket.gethostname(), self.getLocalPort()) try: request = self.requestFilter.fromWire(params) except (TypeError, ValueError, IndexError): return self._makeError('400 Bad Request', "Malformed XMLRPC arguments") rawUrl = self.request.url scheme = self.request.headers.get('X-Conary-Proxy-Target-Scheme') if scheme in ('http', 'https'): rawUrl = str(URL(rawUrl)._replace(scheme=scheme)) # Execution phase -- locate and call the target method try: responseArgs, extraInfo = self.proxyServer.callWrapper( protocol=None, port=None, methodname=method, authToken=self.auth, request=request, remoteIp=self.auth.remote_ip, rawUrl=rawUrl, localAddr=localAddr, protocolString=self.request.http_version, headers=self.request.headers, isSecure=self.isSecure) except errors.InsufficientPermission: return self._makeError('403 Forbidden', "Insufficient permission") rawResponse, headers = responseArgs.toWire(request.version) if extraInfo: headers['Via'] = proxy.formatViaHeader(localAddr, self.request.http_version, prefix=extraInfo.getVia()) response = self.responseFactory(headerlist=headers.items()) response.content_type = 'text/xml' # Output phase -- serialize and write the response body = util.xmlrpcDump((rawResponse,), methodresponse=1) accept = self.request.accept_encoding if len(body) > 200 and 'deflate' in accept: response.content_encoding = 'deflate' response.body = zlib.compress(body, 5) else: response.body = body if (method == 'getChangeSet' and request.version >= 71 and not responseArgs.isException and response.status_int == 200 and responseArgs.result[0] and 'multipart/mixed' in list(self.request.accept) ): return self.inlineChangeset(response, responseArgs, headers) else: return response
def post(port, isSecure, repos, req, authToken=None, repServer=None): if authToken is None: authToken = getAuth(req) if authToken is None: return apache.HTTP_BAD_REQUEST if authToken[0] != "anonymous" and not isSecure and repos.cfg.forceSSL: return apache.HTTP_FORBIDDEN if isSecure: protocol = "https" else: protocol = "http" extraInfo = None repos.log.reset() if req.headers_in['Content-Type'] == "text/xml": # handle XML-RPC requests encoding = req.headers_in.get('Content-Encoding', None) sio = util.BoundedStringIO() try: util.copyStream(req, sio) except IOError, e: # if we got a read timeout, marshal an exception back # to the client print >> sys.stderr, 'error reading from client: %s' % e method = 'unknown - client timeout' response = xmlshims.ResponseArgs.newException( 'ClientTimeout', 'The server was not able to read the ' 'XML-RPC request sent by this client. ' 'This is sometimes caused by MTU problems ' 'on your network connection. Using a ' 'smaller MTU may work around this ' 'problem.') headers = {} startTime = time.time() else: # otherwise, we've read the data, let's process it if encoding == 'deflate': sio.seek(0) try: sio = util.decompressStream(sio) except zlib.error, error: req.log_error("zlib inflate error in POST: %s" % error) return apache.HTTP_BAD_REQUEST startTime = time.time() sio.seek(0) try: (params, method) = util.xmlrpcLoad(sio) except: req.log_error('error parsing XMLRPC request') return apache.HTTP_BAD_REQUEST repos.log(3, "decoding=%s" % method, authToken[0], "%.3f" % (time.time() - startTime)) # req.connection.local_addr[0] is the IP address the server # listens on, not the IP address of the accepted socket. Most of # the times it will be 0.0.0.0 which is not very useful. We're # using local_ip instead, and we grab just the port from # local_addr. localAddr = "%s:%s" % (req.connection.local_ip, req.connection.local_addr[1]) remoteIp = req.connection.remote_ip # Get the IP address of the original request in the case # of a proxy, otherwise use the connection's remote_ip if 'X-Forwarded-For' in req.headers_in: # pick the right-most client, since that is # the one closest to us. For example, if # we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7" # we want to use 4.5.6.7 clients = req.headers_in['X-Forwarded-For'] remoteIp = clients.split(',')[-1].strip() try: request = xmlshims.RequestArgs.fromWire(params) except (TypeError, ValueError, IndexError): req.log_error('error parsing XMLRPC arguments') return apache.HTTP_BAD_REQUEST try: response, extraInfo = repos.callWrapper( protocol=protocol, port=port, methodname=method, authToken=authToken, request=request, remoteIp=remoteIp, rawUrl=req.unparsed_uri, localAddr=localAddr, protocolString=req.protocol, headers=req.headers_in, isSecure=isSecure, ) except errors.InsufficientPermission: return apache.HTTP_FORBIDDEN