def testDecompressStream(self):
data = os.urandom(16 * 1024)
compressed = zlib.compress(data)
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
check = dfo.read()
self.assertEqual(check, data)
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
chunk = dfo.read(333)
self.assertEqual(chunk, data[:333])
# test readline
data = 'hello world\nhello world line 2\n'
compressed = zlib.compress(data)
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
line = dfo.readline()
self.assertEqual(line, 'hello world\n')
line = dfo.readline()
self.assertEqual(line, 'hello world line 2\n')
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
line = dfo.readline(5)
self.assertEqual(line, 'hello')
line = dfo.readline(5)
self.assertEqual(line, ' worl')
line = dfo.readline()
self.assertEqual(line, 'd\n')
def testDecompressStream(self):
data = os.urandom(16 * 1024)
compressed = zlib.compress(data)
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
check = dfo.read()
self.assertEqual(check, data)
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
chunk = dfo.read(333)
self.assertEqual(chunk, data[:333])
# test readline
data = 'hello world\nhello world line 2\n'
compressed = zlib.compress(data)
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
line = dfo.readline()
self.assertEqual(line, 'hello world\n')
line = dfo.readline()
self.assertEqual(line, 'hello world line 2\n')
fp = StringIO.StringIO(compressed)
dfo = util.decompressStream(fp)
line = dfo.readline(5)
self.assertEqual(line, 'hello')
line = dfo.readline(5)
self.assertEqual(line, ' worl')
line = dfo.readline()
self.assertEqual(line, 'd\n')
def _unwrapRPC(request):
stream = request.body_file
encoding = request.headers.get('Content-Encoding', 'identity')
if encoding == 'deflate':
stream = util.decompressStream(stream)
stream.seek(0)
elif encoding != 'identity':
raise exc.HTTPBadRequest("Wrong Content-Encoding for XMLRPC request")
return stream
def _handleResponse(self, req, response):
fp = response
encoding = response.getheader('content-encoding', None)
if encoding == 'deflate':
fp = util.decompressStream(fp)
fp.seek(0)
elif encoding == 'gzip':
fp = util.GzipFile(fileobj=fp)
fp.seek(0)
return ResponseWrapper(fp, response)
def _unwrapRPC(request):
stream = request.body_file
encoding = request.headers.get('Content-Encoding', 'identity')
if encoding == 'deflate':
stream = util.decompressStream(stream)
stream.seek(0)
elif encoding != 'identity':
raise exc.HTTPBadRequest(
"Wrong Content-Encoding for XMLRPC request")
return stream
def _handleResponse(self, req, response):
fp = response
encoding = response.getheader('content-encoding', None)
if encoding == 'deflate':
fp = util.decompressStream(fp)
fp.seek(0)
elif encoding == 'gzip':
fp = util.GzipFile(fileobj=fp)
fp.seek(0)
return ResponseWrapper(fp, response)
def _handleResponse(self, req, response):
fp = response
encoding = response.getheader('content-encoding', None)
if encoding == 'deflate':
# disable until performace is better
#fp = DecompressFileObj(fp)
fp = util.decompressStream(fp)
fp.seek(0)
elif encoding == 'gzip':
fp = util.GzipFile(fileobj=fp)
fp.seek(0)
return ResponseWrapper(fp, response)
def _handleResponse(self, req, response):
fp = response
encoding = response.getheader('content-encoding', None)
if encoding == 'deflate':
# disable until performace is better
#fp = DecompressFileObj(fp)
fp = util.decompressStream(fp)
fp.seek(0)
elif encoding == 'gzip':
fp = util.GzipFile(fileobj=fp)
fp.seek(0)
return ResponseWrapper(fp, response)
def rpcHandler(context):
req = context.req
# only handle POSTs
if req.method.upper() != 'POST':
raise web_exc.HTTPMethodNotAllowed(allow='POST')
if req.content_type == 'text/xml':
kind = 'xml'
elif req.content_type == 'application/x-json':
kind = 'json'
else:
raise web_exc.HTTPBadRequest()
# instantiate a MintServer
srvr = server.MintServer(context.cfg, allowPrivate=True, req=req,
db=context.db)
stream = req.body_file
encoding = req.headers.get('Content-Encoding', 'identity')
if encoding == 'deflate':
stream = util.decompressStream(stream)
stream.seek(0)
elif encoding != 'identity':
raise web_exc.HTTPBadRequest()
if kind == 'xml':
(args, method) = util.xmlrpcLoad(stream)
else:
(method, args) = json.load(stream)
# coax parameters into something MintServer likes
params = [method, context.authToken, args]
# go for it; return 403 if permission is denied
try:
# result is (isError, returnValues)
result = srvr.callWrapper(*params)
except (errors.InsufficientPermission, mint_error.PermissionDenied):
return context.responseFactory(
"<h1>Forbidden</h1>\n"
"<p>Access denied by the server.</p>\n",
status='403 Forbidden',
content_type='text/html')
# create a response
if kind == 'xml':
resp = xmlrpclib.dumps((result,), methodresponse=1)
else:
resp = json.dumps(result[1])
return context.responseFactory(content_type=req.content_type, body=resp)
def testCompressDecompressStream(self):
# Test that compressing and uncompressing streams produces the same
# data
fd, tempf = tempfile.mkstemp()
os.unlink(tempf)
sio = os.fdopen(fd, "w+")
# Some data we will compress
for fn in ['distcache-1.4.5-2.src.rpm', 'distcc-2.9.tar.bz2',
'initscripts-10-11.src.rpm', 'jcd.iso']:
util.copyStream(file(os.path.join(resources.get_archive(), fn)), sio)
sio.seek(0)
cstr = util.compressStream(sio)
cstr.seek(0)
dstr = util.decompressStream(cstr)
dstr.seek(0)
sio.seek(0)
self.assertEqual(sio.read(), dstr.read())
def testCompressDecompressStream(self):
# Test that compressing and uncompressing streams produces the same
# data
fd, tempf = tempfile.mkstemp()
os.unlink(tempf)
sio = os.fdopen(fd, "w+")
# Some data we will compress
for fn in ['distcache-1.4.5-2.src.rpm', 'distcc-2.9.tar.bz2',
'initscripts-10-11.src.rpm', 'jcd.iso']:
util.copyStream(file(os.path.join(resources.get_archive(), fn)), sio)
sio.seek(0)
cstr = util.compressStream(sio)
cstr.seek(0)
dstr = util.decompressStream(cstr)
dstr.seek(0)
sio.seek(0)
self.assertEqual(sio.read(), dstr.read())
def postRpc(self):
if self.request.content_type != 'text/xml':
return self._makeError('400 Bad Request',
"Unrecognized Content-Type")
stream = self.request.body_file
encoding = self.request.headers.get('Content-Encoding', 'identity')
if encoding == 'deflate':
stream = util.decompressStream(stream)
stream.seek(0)
elif encoding != 'identity':
return self._makeError('400 Bad Request',
"Unrecognized Content-Encoding")
try:
params, method = util.xmlrpcLoad(stream)
except:
return self._makeError('400 Bad Request',
"Malformed XMLRPC request")
localAddr = '%s:%s' % (socket.gethostname(), self.getLocalPort())
try:
request = self.requestFilter.fromWire(params)
except (TypeError, ValueError, IndexError):
return self._makeError('400 Bad Request',
"Malformed XMLRPC arguments")
rawUrl = self.request.url
scheme = self.request.headers.get('X-Conary-Proxy-Target-Scheme')
if scheme in ('http', 'https'):
rawUrl = str(URL(rawUrl)._replace(scheme=scheme))
# Execution phase -- locate and call the target method
try:
responseArgs, extraInfo = self.proxyServer.callWrapper(
protocol=None,
port=None,
methodname=method,
authToken=self.auth,
request=request,
remoteIp=self.auth.remote_ip,
rawUrl=rawUrl,
localAddr=localAddr,
protocolString=self.request.http_version,
headers=self.request.headers,
isSecure=self.isSecure)
except errors.InsufficientPermission:
return self._makeError('403 Forbidden', "Insufficient permission")
rawResponse, headers = responseArgs.toWire(request.version)
if extraInfo:
headers['Via'] = proxy.formatViaHeader(localAddr,
self.request.http_version,
prefix=extraInfo.getVia())
response = self.responseFactory(headerlist=headers.items())
response.content_type = 'text/xml'
# Output phase -- serialize and write the response
body = util.xmlrpcDump((rawResponse, ), methodresponse=1)
accept = self.request.accept_encoding
if len(body) > 200 and 'deflate' in accept:
response.content_encoding = 'deflate'
response.body = zlib.compress(body, 5)
else:
response.body = body
if (method == 'getChangeSet' and request.version >= 71
and not responseArgs.isException and response.status_int == 200
and responseArgs.result[0]
and 'multipart/mixed' in list(self.request.accept)):
return self.inlineChangeset(response, responseArgs, headers)
else:
return response
def handleXml(self, authToken):
contentLength = int(self.headers['Content-Length'])
sio = util.BoundedStringIO()
actual = util.copyStream(self.rfile, sio, contentLength)
if contentLength != actual:
raise Exception(contentLength, actual)
sio.seek(0)
encoding = self.headers.get('Content-Encoding', None)
if encoding == 'deflate':
sio = util.decompressStream(sio)
sio.seek(0)
(params, method) = util.xmlrpcLoad(sio)
logMe(3, "decoded xml-rpc call %s from %d bytes request" %(method, contentLength))
if self.netProxy:
repos = self.netProxy
else:
repos = self.netRepos
localHost, localPort = self.request.getsockname()[:2]
if ':' in localHost:
localHost = '[%s]' % localHost
localAddr = '%s:%s' % (localHost, localPort)
request = xmlshims.RequestArgs.fromWire(params)
if repos is not None:
try:
response, extraInfo = repos.callWrapper(
protocol='http',
port=None,
methodname=method,
authToken=authToken,
request=request,
remoteIp=self.connection.getpeername()[0],
rawUrl=self.path,
localAddr=localAddr,
protocolString=self.request_version,
headers=self.headers,
isSecure=self.server.isSecure,
)
except errors.InsufficientPermission:
self.send_error(403)
return None
except:
# exceptions are handled (logged) in callWrapper - send
# 500 code back to the client to indicate an error happened
self.send_error(500)
from conary.lib import formattrace
excType, excValue, excTb = sys.exc_info()
formattrace.formatTrace(excType, excValue, excTb,
withLocals = False)
return None
logMe(3, "returned from", method)
rawResponse, headers = response.toWire(request.version)
sio = util.BoundedStringIO()
util.xmlrpcDump((rawResponse,), stream = sio, methodresponse=1)
respLen = sio.tell()
logMe(3, "encoded xml-rpc response to %d bytes" % respLen)
self.send_response(200)
encoding = self.headers.get('Accept-encoding', '')
if respLen > 200 and 'deflate' in encoding:
sio.seek(0)
sio = util.compressStream(sio, level = 5)
respLen = sio.tell()
self.send_header('Content-encoding', 'deflate')
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(respLen))
for key, value in sorted(headers.items()):
self.send_header(key, value)
if extraInfo:
# If available, send to the client the via headers all the way up
# to us
self.send_header('Via', proxy.formatViaHeader(localAddr,
'HTTP/1.0', prefix=extraInfo.getVia()))
self.end_headers()
sio.seek(0)
util.copyStream(sio, self.wfile)
logMe(3, "sent response to client", respLen, "bytes")
return respLen
def post(port, isSecure, repos, req, authToken=None, repServer=None):
if authToken is None:
authToken = getAuth(req)
if authToken is None:
return apache.HTTP_BAD_REQUEST
if authToken[0] != "anonymous" and not isSecure and repos.cfg.forceSSL:
return apache.HTTP_FORBIDDEN
if isSecure:
protocol = "https"
else:
protocol = "http"
extraInfo = None
repos.log.reset()
if req.headers_in['Content-Type'] == "text/xml":
# handle XML-RPC requests
encoding = req.headers_in.get('Content-Encoding', None)
sio = util.BoundedStringIO()
try:
util.copyStream(req, sio)
except IOError, e:
# if we got a read timeout, marshal an exception back
# to the client
print >> sys.stderr, 'error reading from client: %s' %e
method = 'unknown - client timeout'
response = xmlshims.ResponseArgs.newException('ClientTimeout',
'The server was not able to read the '
'XML-RPC request sent by this client. '
'This is sometimes caused by MTU problems '
'on your network connection. Using a '
'smaller MTU may work around this '
'problem.')
headers = {}
startTime = time.time()
else:
# otherwise, we've read the data, let's process it
if encoding == 'deflate':
sio.seek(0)
try:
sio = util.decompressStream(sio)
except zlib.error, error:
req.log_error("zlib inflate error in POST: %s" % error)
return apache.HTTP_BAD_REQUEST
startTime = time.time()
sio.seek(0)
try:
(params, method) = util.xmlrpcLoad(sio)
except:
req.log_error('error parsing XMLRPC request')
return apache.HTTP_BAD_REQUEST
repos.log(3, "decoding=%s" % method, authToken[0],
"%.3f" % (time.time()-startTime))
# req.connection.local_addr[0] is the IP address the server
# listens on, not the IP address of the accepted socket. Most of
# the times it will be 0.0.0.0 which is not very useful. We're
# using local_ip instead, and we grab just the port from
# local_addr.
localAddr = "%s:%s" % (req.connection.local_ip,
req.connection.local_addr[1])
remoteIp = req.connection.remote_ip
# Get the IP address of the original request in the case
# of a proxy, otherwise use the connection's remote_ip
if 'X-Forwarded-For' in req.headers_in:
# pick the right-most client, since that is
# the one closest to us. For example, if
# we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7"
# we want to use 4.5.6.7
clients = req.headers_in['X-Forwarded-For']
remoteIp = clients.split(',')[-1].strip()
try:
request = xmlshims.RequestArgs.fromWire(params)
except (TypeError, ValueError, IndexError):
req.log_error('error parsing XMLRPC arguments')
return apache.HTTP_BAD_REQUEST
try:
response, extraInfo = repos.callWrapper(
protocol=protocol,
port=port,
methodname=method,
authToken=authToken,
request=request,
remoteIp=remoteIp,
rawUrl=req.unparsed_uri,
localAddr=localAddr,
protocolString=req.protocol,
headers=req.headers_in,
isSecure=isSecure,
)
except errors.InsufficientPermission:
return apache.HTTP_FORBIDDEN
def postRpc(self):
if self.request.content_type != 'text/xml':
return self._makeError('400 Bad Request',
"Unrecognized Content-Type")
stream = self.request.body_file
encoding = self.request.headers.get('Content-Encoding', 'identity')
if encoding == 'deflate':
stream = util.decompressStream(stream)
stream.seek(0)
elif encoding != 'identity':
return self._makeError('400 Bad Request',
"Unrecognized Content-Encoding")
try:
params, method = util.xmlrpcLoad(stream)
except:
return self._makeError('400 Bad Request',
"Malformed XMLRPC request")
localAddr = socket.gethostname()
try:
request = self.requestFilter.fromWire(params)
except (TypeError, ValueError, IndexError):
return self._makeError('400 Bad Request',
"Malformed XMLRPC arguments")
# Execution phase -- locate and call the target method
try:
response, extraInfo = self.proxyServer.callWrapper(
protocol=None,
port=None,
methodname=method,
authToken=self.auth,
request=request,
remoteIp=self.auth.remote_ip,
rawUrl=self.request.url,
localAddr=localAddr,
protocolString=self.request.http_version,
headers=self.request.headers,
isSecure=self.isSecure)
except errors.InsufficientPermission:
return self._makeError('403 Forbidden', "Insufficient permission")
rawResponse, headers = response.toWire(request.version)
response = self.responseFactory(
headerlist=headers.items(),
content_type='text/xml',
)
# Output phase -- serialize and write the response
body = util.xmlrpcDump((rawResponse, ), methodresponse=1)
accept = self.request.accept_encoding
if len(body) > 200 and 'deflate' in accept:
response.content_encoding = 'deflate'
response.body = zlib.compress(body, 5)
else:
response.body = body
if extraInfo:
headers['Via'] = proxy.formatViaHeader(localAddr,
self.request.http_version,
prefix=extraInfo.getVia())
return response
def handleXml(self, authToken):
contentLength = int(self.headers['Content-Length'])
sio = util.BoundedStringIO()
actual = util.copyStream(self.rfile, sio, contentLength)
if contentLength != actual:
raise Exception(contentLength, actual)
sio.seek(0)
encoding = self.headers.get('Content-Encoding', None)
if encoding == 'deflate':
sio = util.decompressStream(sio)
sio.seek(0)
(params, method) = util.xmlrpcLoad(sio)
logMe(
3, "decoded xml-rpc call %s from %d bytes request" %
(method, contentLength))
if self.netProxy:
repos = self.netProxy
else:
repos = self.netRepos
localHost, localPort = self.request.getsockname()[:2]
if ':' in localHost:
localHost = '[%s]' % localHost
localAddr = '%s:%s' % (localHost, localPort)
request = xmlshims.RequestArgs.fromWire(params)
if repos is not None:
try:
response, extraInfo = repos.callWrapper(
protocol='http',
port=None,
methodname=method,
authToken=authToken,
request=request,
remoteIp=self.connection.getpeername()[0],
rawUrl=self.path,
localAddr=localAddr,
protocolString=self.request_version,
headers=self.headers,
isSecure=self.server.isSecure,
)
except errors.InsufficientPermission:
self.send_error(403)
return None
except:
# exceptions are handled (logged) in callWrapper - send
# 500 code back to the client to indicate an error happened
self.send_error(500)
from conary.lib import formattrace
excType, excValue, excTb = sys.exc_info()
formattrace.formatTrace(excType,
excValue,
excTb,
withLocals=False)
return None
logMe(3, "returned from", method)
rawResponse, headers = response.toWire(request.version)
sio = util.BoundedStringIO()
util.xmlrpcDump((rawResponse, ), stream=sio, methodresponse=1)
respLen = sio.tell()
logMe(3, "encoded xml-rpc response to %d bytes" % respLen)
self.send_response(200)
encoding = self.headers.get('Accept-encoding', '')
if respLen > 200 and 'deflate' in encoding:
sio.seek(0)
sio = util.compressStream(sio, level=5)
respLen = sio.tell()
self.send_header('Content-encoding', 'deflate')
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(respLen))
for key, value in sorted(headers.items()):
self.send_header(key, value)
if extraInfo:
# If available, send to the client the via headers all the way up
# to us
self.send_header(
'Via',
proxy.formatViaHeader(localAddr,
'HTTP/1.0',
prefix=extraInfo.getVia()))
self.end_headers()
sio.seek(0)
util.copyStream(sio, self.wfile)
logMe(3, "sent response to client", respLen, "bytes")
return respLen
def postRpc(self):
if self.request.content_type != 'text/xml':
return self._makeError('400 Bad Request',
"Unrecognized Content-Type")
stream = self.request.body_file
encoding = self.request.headers.get('Content-Encoding', 'identity')
if encoding == 'deflate':
stream = util.decompressStream(stream)
stream.seek(0)
elif encoding != 'identity':
return self._makeError('400 Bad Request',
"Unrecognized Content-Encoding")
try:
params, method = util.xmlrpcLoad(stream)
except:
return self._makeError('400 Bad Request',
"Malformed XMLRPC request")
localAddr = '%s:%s' % (socket.gethostname(), self.getLocalPort())
try:
request = self.requestFilter.fromWire(params)
except (TypeError, ValueError, IndexError):
return self._makeError('400 Bad Request',
"Malformed XMLRPC arguments")
rawUrl = self.request.url
scheme = self.request.headers.get('X-Conary-Proxy-Target-Scheme')
if scheme in ('http', 'https'):
rawUrl = str(URL(rawUrl)._replace(scheme=scheme))
# Execution phase -- locate and call the target method
try:
responseArgs, extraInfo = self.proxyServer.callWrapper(
protocol=None,
port=None,
methodname=method,
authToken=self.auth,
request=request,
remoteIp=self.auth.remote_ip,
rawUrl=rawUrl,
localAddr=localAddr,
protocolString=self.request.http_version,
headers=self.request.headers,
isSecure=self.isSecure)
except errors.InsufficientPermission:
return self._makeError('403 Forbidden', "Insufficient permission")
rawResponse, headers = responseArgs.toWire(request.version)
if extraInfo:
headers['Via'] = proxy.formatViaHeader(localAddr,
self.request.http_version, prefix=extraInfo.getVia())
response = self.responseFactory(headerlist=headers.items())
response.content_type = 'text/xml'
# Output phase -- serialize and write the response
body = util.xmlrpcDump((rawResponse,), methodresponse=1)
accept = self.request.accept_encoding
if len(body) > 200 and 'deflate' in accept:
response.content_encoding = 'deflate'
response.body = zlib.compress(body, 5)
else:
response.body = body
if (method == 'getChangeSet'
and request.version >= 71
and not responseArgs.isException
and response.status_int == 200
and responseArgs.result[0]
and 'multipart/mixed' in list(self.request.accept)
):
return self.inlineChangeset(response, responseArgs, headers)
else:
return response
def post(port, isSecure, repos, req, authToken=None, repServer=None):
if authToken is None:
authToken = getAuth(req)
if authToken is None:
return apache.HTTP_BAD_REQUEST
if authToken[0] != "anonymous" and not isSecure and repos.cfg.forceSSL:
return apache.HTTP_FORBIDDEN
if isSecure:
protocol = "https"
else:
protocol = "http"
extraInfo = None
repos.log.reset()
if req.headers_in['Content-Type'] == "text/xml":
# handle XML-RPC requests
encoding = req.headers_in.get('Content-Encoding', None)
sio = util.BoundedStringIO()
try:
util.copyStream(req, sio)
except IOError, e:
# if we got a read timeout, marshal an exception back
# to the client
print >> sys.stderr, 'error reading from client: %s' % e
method = 'unknown - client timeout'
response = xmlshims.ResponseArgs.newException(
'ClientTimeout', 'The server was not able to read the '
'XML-RPC request sent by this client. '
'This is sometimes caused by MTU problems '
'on your network connection. Using a '
'smaller MTU may work around this '
'problem.')
headers = {}
startTime = time.time()
else:
# otherwise, we've read the data, let's process it
if encoding == 'deflate':
sio.seek(0)
try:
sio = util.decompressStream(sio)
except zlib.error, error:
req.log_error("zlib inflate error in POST: %s" % error)
return apache.HTTP_BAD_REQUEST
startTime = time.time()
sio.seek(0)
try:
(params, method) = util.xmlrpcLoad(sio)
except:
req.log_error('error parsing XMLRPC request')
return apache.HTTP_BAD_REQUEST
repos.log(3, "decoding=%s" % method, authToken[0],
"%.3f" % (time.time() - startTime))
# req.connection.local_addr[0] is the IP address the server
# listens on, not the IP address of the accepted socket. Most of
# the times it will be 0.0.0.0 which is not very useful. We're
# using local_ip instead, and we grab just the port from
# local_addr.
localAddr = "%s:%s" % (req.connection.local_ip,
req.connection.local_addr[1])
remoteIp = req.connection.remote_ip
# Get the IP address of the original request in the case
# of a proxy, otherwise use the connection's remote_ip
if 'X-Forwarded-For' in req.headers_in:
# pick the right-most client, since that is
# the one closest to us. For example, if
# we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7"
# we want to use 4.5.6.7
clients = req.headers_in['X-Forwarded-For']
remoteIp = clients.split(',')[-1].strip()
try:
request = xmlshims.RequestArgs.fromWire(params)
except (TypeError, ValueError, IndexError):
req.log_error('error parsing XMLRPC arguments')
return apache.HTTP_BAD_REQUEST
try:
response, extraInfo = repos.callWrapper(
protocol=protocol,
port=port,
methodname=method,
authToken=authToken,
request=request,
remoteIp=remoteIp,
rawUrl=req.unparsed_uri,
localAddr=localAddr,
protocolString=req.protocol,
headers=req.headers_in,
isSecure=isSecure,
)
except errors.InsufficientPermission:
return apache.HTTP_FORBIDDEN