def testCompressDecompressStream(self):
# Test that compressing and uncompressing streams produces the same
# data
fd, tempf = tempfile.mkstemp()
os.unlink(tempf)
sio = os.fdopen(fd, "w+")
# Some data we will compress
for fn in ['distcache-1.4.5-2.src.rpm', 'distcc-2.9.tar.bz2',
'initscripts-10-11.src.rpm', 'jcd.iso']:
util.copyStream(file(os.path.join(resources.get_archive(), fn)), sio)
sio.seek(0)
cstr = util.compressStream(sio)
cstr.seek(0)
dstr = util.decompressStream(cstr)
dstr.seek(0)
sio.seek(0)
self.assertEqual(sio.read(), dstr.read())
def testCompressDecompressStream(self):
# Test that compressing and uncompressing streams produces the same
# data
fd, tempf = tempfile.mkstemp()
os.unlink(tempf)
sio = os.fdopen(fd, "w+")
# Some data we will compress
for fn in ['distcache-1.4.5-2.src.rpm', 'distcc-2.9.tar.bz2',
'initscripts-10-11.src.rpm', 'jcd.iso']:
util.copyStream(file(os.path.join(resources.get_archive(), fn)), sio)
sio.seek(0)
cstr = util.compressStream(sio)
cstr.seek(0)
dstr = util.decompressStream(cstr)
dstr.seek(0)
sio.seek(0)
self.assertEqual(sio.read(), dstr.read())
def handleXml(self, authToken):
contentLength = int(self.headers['Content-Length'])
sio = util.BoundedStringIO()
actual = util.copyStream(self.rfile, sio, contentLength)
if contentLength != actual:
raise Exception(contentLength, actual)
sio.seek(0)
encoding = self.headers.get('Content-Encoding', None)
if encoding == 'deflate':
sio = util.decompressStream(sio)
sio.seek(0)
(params, method) = util.xmlrpcLoad(sio)
logMe(
3, "decoded xml-rpc call %s from %d bytes request" %
(method, contentLength))
if self.netProxy:
repos = self.netProxy
else:
repos = self.netRepos
localHost, localPort = self.request.getsockname()[:2]
if ':' in localHost:
localHost = '[%s]' % localHost
localAddr = '%s:%s' % (localHost, localPort)
request = xmlshims.RequestArgs.fromWire(params)
if repos is not None:
try:
response, extraInfo = repos.callWrapper(
protocol='http',
port=None,
methodname=method,
authToken=authToken,
request=request,
remoteIp=self.connection.getpeername()[0],
rawUrl=self.path,
localAddr=localAddr,
protocolString=self.request_version,
headers=self.headers,
isSecure=self.server.isSecure,
)
except errors.InsufficientPermission:
self.send_error(403)
return None
except:
# exceptions are handled (logged) in callWrapper - send
# 500 code back to the client to indicate an error happened
self.send_error(500)
from conary.lib import formattrace
excType, excValue, excTb = sys.exc_info()
formattrace.formatTrace(excType,
excValue,
excTb,
withLocals=False)
return None
logMe(3, "returned from", method)
rawResponse, headers = response.toWire(request.version)
sio = util.BoundedStringIO()
util.xmlrpcDump((rawResponse, ), stream=sio, methodresponse=1)
respLen = sio.tell()
logMe(3, "encoded xml-rpc response to %d bytes" % respLen)
self.send_response(200)
encoding = self.headers.get('Accept-encoding', '')
if respLen > 200 and 'deflate' in encoding:
sio.seek(0)
sio = util.compressStream(sio, level=5)
respLen = sio.tell()
self.send_header('Content-encoding', 'deflate')
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(respLen))
for key, value in sorted(headers.items()):
self.send_header(key, value)
if extraInfo:
# If available, send to the client the via headers all the way up
# to us
self.send_header(
'Via',
proxy.formatViaHeader(localAddr,
'HTTP/1.0',
prefix=extraInfo.getVia()))
self.end_headers()
sio.seek(0)
util.copyStream(sio, self.wfile)
logMe(3, "sent response to client", respLen, "bytes")
return respLen
def post(port, isSecure, repos, req, authToken=None, repServer=None):
if authToken is None:
authToken = getAuth(req)
if authToken is None:
return apache.HTTP_BAD_REQUEST
if authToken[0] != "anonymous" and not isSecure and repos.cfg.forceSSL:
return apache.HTTP_FORBIDDEN
if isSecure:
protocol = "https"
else:
protocol = "http"
extraInfo = None
repos.log.reset()
if req.headers_in['Content-Type'] == "text/xml":
# handle XML-RPC requests
encoding = req.headers_in.get('Content-Encoding', None)
sio = util.BoundedStringIO()
try:
util.copyStream(req, sio)
except IOError, e:
# if we got a read timeout, marshal an exception back
# to the client
print >> sys.stderr, 'error reading from client: %s' % e
method = 'unknown - client timeout'
response = xmlshims.ResponseArgs.newException(
'ClientTimeout', 'The server was not able to read the '
'XML-RPC request sent by this client. '
'This is sometimes caused by MTU problems '
'on your network connection. Using a '
'smaller MTU may work around this '
'problem.')
headers = {}
startTime = time.time()
else:
# otherwise, we've read the data, let's process it
if encoding == 'deflate':
sio.seek(0)
try:
sio = util.decompressStream(sio)
except zlib.error, error:
req.log_error("zlib inflate error in POST: %s" % error)
return apache.HTTP_BAD_REQUEST
startTime = time.time()
sio.seek(0)
try:
(params, method) = util.xmlrpcLoad(sio)
except:
req.log_error('error parsing XMLRPC request')
return apache.HTTP_BAD_REQUEST
repos.log(3, "decoding=%s" % method, authToken[0],
"%.3f" % (time.time() - startTime))
# req.connection.local_addr[0] is the IP address the server
# listens on, not the IP address of the accepted socket. Most of
# the times it will be 0.0.0.0 which is not very useful. We're
# using local_ip instead, and we grab just the port from
# local_addr.
localAddr = "%s:%s" % (req.connection.local_ip,
req.connection.local_addr[1])
remoteIp = req.connection.remote_ip
# Get the IP address of the original request in the case
# of a proxy, otherwise use the connection's remote_ip
if 'X-Forwarded-For' in req.headers_in:
# pick the right-most client, since that is
# the one closest to us. For example, if
# we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7"
# we want to use 4.5.6.7
clients = req.headers_in['X-Forwarded-For']
remoteIp = clients.split(',')[-1].strip()
try:
request = xmlshims.RequestArgs.fromWire(params)
except (TypeError, ValueError, IndexError):
req.log_error('error parsing XMLRPC arguments')
return apache.HTTP_BAD_REQUEST
try:
response, extraInfo = repos.callWrapper(
protocol=protocol,
port=port,
methodname=method,
authToken=authToken,
request=request,
remoteIp=remoteIp,
rawUrl=req.unparsed_uri,
localAddr=localAddr,
protocolString=req.protocol,
headers=req.headers_in,
isSecure=isSecure,
)
except errors.InsufficientPermission:
return apache.HTTP_FORBIDDEN
if respLen > 200 and 'deflate' in encoding:
req.headers_out['Content-encoding'] = 'deflate'
sio.seek(0)
sio = util.compressStream(sio, 5)
respLen = sio.tell()
req.headers_out['Content-length'] = '%d' % respLen
for key, value in sorted(headers.items()):
req.headers_out[key] = value
if extraInfo:
# If available, send to the client the via headers all the way up
# to us
req.headers_out['Via'] = proxy.formatViaHeader(
localAddr, 'HTTP/1.0', prefix=extraInfo.getVia())
sio.seek(0)
util.copyStream(sio, req)
return apache.OK
else:
# Handle HTTP (web browser) requests
from conary.server import wsgi_adapter
from conary.web import repos_web
httpHandler = repos_web.ReposWeb(repos.cfg, repositoryServer=repServer)
return wsgi_adapter.modpython_to_webob(req, httpHandler._handleRequest)
def sendfile(req, size, path):
# FIXME: apache 2.0 can't sendfile() a file > 2 GiB.
# we'll have to send the data ourselves
if size >= 0x80000000:
f = open(path, 'r')
# 2 MB buffer
def handleXml(self, authToken):
contentLength = int(self.headers['Content-Length'])
sio = util.BoundedStringIO()
actual = util.copyStream(self.rfile, sio, contentLength)
if contentLength != actual:
raise Exception(contentLength, actual)
sio.seek(0)
encoding = self.headers.get('Content-Encoding', None)
if encoding == 'deflate':
sio = util.decompressStream(sio)
sio.seek(0)
(params, method) = util.xmlrpcLoad(sio)
logMe(3, "decoded xml-rpc call %s from %d bytes request" %(method, contentLength))
if self.netProxy:
repos = self.netProxy
else:
repos = self.netRepos
localHost, localPort = self.request.getsockname()[:2]
if ':' in localHost:
localHost = '[%s]' % localHost
localAddr = '%s:%s' % (localHost, localPort)
request = xmlshims.RequestArgs.fromWire(params)
if repos is not None:
try:
response, extraInfo = repos.callWrapper(
protocol='http',
port=None,
methodname=method,
authToken=authToken,
request=request,
remoteIp=self.connection.getpeername()[0],
rawUrl=self.path,
localAddr=localAddr,
protocolString=self.request_version,
headers=self.headers,
isSecure=self.server.isSecure,
)
except errors.InsufficientPermission:
self.send_error(403)
return None
except:
# exceptions are handled (logged) in callWrapper - send
# 500 code back to the client to indicate an error happened
self.send_error(500)
from conary.lib import formattrace
excType, excValue, excTb = sys.exc_info()
formattrace.formatTrace(excType, excValue, excTb,
withLocals = False)
return None
logMe(3, "returned from", method)
rawResponse, headers = response.toWire(request.version)
sio = util.BoundedStringIO()
util.xmlrpcDump((rawResponse,), stream = sio, methodresponse=1)
respLen = sio.tell()
logMe(3, "encoded xml-rpc response to %d bytes" % respLen)
self.send_response(200)
encoding = self.headers.get('Accept-encoding', '')
if respLen > 200 and 'deflate' in encoding:
sio.seek(0)
sio = util.compressStream(sio, level = 5)
respLen = sio.tell()
self.send_header('Content-encoding', 'deflate')
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(respLen))
for key, value in sorted(headers.items()):
self.send_header(key, value)
if extraInfo:
# If available, send to the client the via headers all the way up
# to us
self.send_header('Via', proxy.formatViaHeader(localAddr,
'HTTP/1.0', prefix=extraInfo.getVia()))
self.end_headers()
sio.seek(0)
util.copyStream(sio, self.wfile)
logMe(3, "sent response to client", respLen, "bytes")
return respLen
def post(port, isSecure, repos, req, authToken=None, repServer=None):
if authToken is None:
authToken = getAuth(req)
if authToken is None:
return apache.HTTP_BAD_REQUEST
if authToken[0] != "anonymous" and not isSecure and repos.cfg.forceSSL:
return apache.HTTP_FORBIDDEN
if isSecure:
protocol = "https"
else:
protocol = "http"
extraInfo = None
repos.log.reset()
if req.headers_in['Content-Type'] == "text/xml":
# handle XML-RPC requests
encoding = req.headers_in.get('Content-Encoding', None)
sio = util.BoundedStringIO()
try:
util.copyStream(req, sio)
except IOError, e:
# if we got a read timeout, marshal an exception back
# to the client
print >> sys.stderr, 'error reading from client: %s' %e
method = 'unknown - client timeout'
response = xmlshims.ResponseArgs.newException('ClientTimeout',
'The server was not able to read the '
'XML-RPC request sent by this client. '
'This is sometimes caused by MTU problems '
'on your network connection. Using a '
'smaller MTU may work around this '
'problem.')
headers = {}
startTime = time.time()
else:
# otherwise, we've read the data, let's process it
if encoding == 'deflate':
sio.seek(0)
try:
sio = util.decompressStream(sio)
except zlib.error, error:
req.log_error("zlib inflate error in POST: %s" % error)
return apache.HTTP_BAD_REQUEST
startTime = time.time()
sio.seek(0)
try:
(params, method) = util.xmlrpcLoad(sio)
except:
req.log_error('error parsing XMLRPC request')
return apache.HTTP_BAD_REQUEST
repos.log(3, "decoding=%s" % method, authToken[0],
"%.3f" % (time.time()-startTime))
# req.connection.local_addr[0] is the IP address the server
# listens on, not the IP address of the accepted socket. Most of
# the times it will be 0.0.0.0 which is not very useful. We're
# using local_ip instead, and we grab just the port from
# local_addr.
localAddr = "%s:%s" % (req.connection.local_ip,
req.connection.local_addr[1])
remoteIp = req.connection.remote_ip
# Get the IP address of the original request in the case
# of a proxy, otherwise use the connection's remote_ip
if 'X-Forwarded-For' in req.headers_in:
# pick the right-most client, since that is
# the one closest to us. For example, if
# we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7"
# we want to use 4.5.6.7
clients = req.headers_in['X-Forwarded-For']
remoteIp = clients.split(',')[-1].strip()
try:
request = xmlshims.RequestArgs.fromWire(params)
except (TypeError, ValueError, IndexError):
req.log_error('error parsing XMLRPC arguments')
return apache.HTTP_BAD_REQUEST
try:
response, extraInfo = repos.callWrapper(
protocol=protocol,
port=port,
methodname=method,
authToken=authToken,
request=request,
remoteIp=remoteIp,
rawUrl=req.unparsed_uri,
localAddr=localAddr,
protocolString=req.protocol,
headers=req.headers_in,
isSecure=isSecure,
)
except errors.InsufficientPermission:
return apache.HTTP_FORBIDDEN
if respLen > 200 and 'deflate' in encoding:
req.headers_out['Content-encoding'] = 'deflate'
sio.seek(0)
sio = util.compressStream(sio, 5)
respLen = sio.tell()
req.headers_out['Content-length'] = '%d' % respLen
for key, value in sorted(headers.items()):
req.headers_out[key] = value
if extraInfo:
# If available, send to the client the via headers all the way up
# to us
req.headers_out['Via'] = proxy.formatViaHeader(localAddr,
'HTTP/1.0', prefix=extraInfo.getVia())
sio.seek(0)
util.copyStream(sio, req)
return apache.OK
else:
# Handle HTTP (web browser) requests
from conary.server import wsgi_adapter
from conary.web import repos_web
httpHandler = repos_web.ReposWeb(repos.cfg, repositoryServer=repServer)
return wsgi_adapter.modpython_to_webob(req, httpHandler._handleRequest)
def sendfile(req, size, path):
# FIXME: apache 2.0 can't sendfile() a file > 2 GiB.
# we'll have to send the data ourselves
if size >= 0x80000000:
f = open(path, 'r')
# 2 MB buffer