def read(self, fd=sys.stdin):
# The octet-streams are passed right back
if self.type == "application/octet-stream":
return
if self.length:
# Read exactly the amount of data we were told
self.io = _smart_read(fd,
self.length,
bufferSize=self.bufferSize,
progressCallback=self.progressCallback,
max_mem_size=self.max_mem_size)
else:
# Oh well, no clue; read until EOF (hopefully)
self.io = _smart_total_read(fd)
if not self.transfer or self.transfer == "binary":
return
elif self.transfer == "base64":
import base64
old_io = self.io
old_io.seek(0, 0)
self.io = SmartIO(max_mem_size=self.max_mem_size)
base64.decode(old_io, self.io)
else:
raise NotImplementedError(self.transfer)
def _smart_read(fd, amt, bufferSize=1024, progressCallback=None,
max_mem_size=16384):
# Reads amt bytes from fd, or until the end of file, whichever
# occurs first
# The function will read in memory if the amout to be read is smaller than
# max_mem_size, or to a temporary file otherwise
#
# Unlike read(), _smart_read tries to return exactly the requested amount
# (whereas read will return _up_to_ that amount). Reads from sockets will
# usually reaturn less data, or the read can be interrupted
#
# Inspired by Greg Stein's httplib.py (the standard in python 2.x)
#
# support for progress callbacks added
startTime = time.time()
lastTime = startTime
buf = SmartIO(max_mem_size=max_mem_size)
origsize = amt
while amt > 0:
curTime = time.time()
l = min(bufferSize, amt)
chunk = fd.read(l)
# read guarantees that len(chunk) <= l
l = len(chunk)
if not l:
# Oops. Most likely EOF
break
# And since the original l was smaller than amt, we know amt >= 0
amt = amt - l
buf.write(chunk)
if progressCallback is None:
# No progress callback, so don't do fancy computations
continue
# We update the progress callback if:
# we haven't updated it for more than a secord, or
# it's the last read (amt == 0)
if curTime - lastTime >= 1 or amt == 0:
lastTime = curTime
# use float() so that we force float division in the next step
bytesRead = float(origsize - amt)
# if amt == 0, on a fast machine it is possible to have
# curTime - lastTime == 0, so add an epsilon to prevent a division
# by zero
speed = bytesRead / ((curTime - startTime) + .000001)
if origsize == 0:
secs = 0
else:
# speed != 0 because bytesRead > 0
# (if bytesRead == 0 then origsize == amt, which means a read
# of 0 length; but that's impossible since we already checked
# that l is non-null
secs = amt / speed
progressCallback(bytesRead, origsize, speed, secs)
# Now rewind the SmartIO
buf.seek(0, 0)
return buf
def process(self, data):
# Assume straight text/xml
self.data = data
# Content-Encoding header
if self.encoding == self.ENCODE_GZIP:
import gzip
encoding_name = self.encodings[self.ENCODE_GZIP][0]
self.set_header("Content-Encoding", encoding_name)
f = SmartIO(force_mem=1)
gz = gzip.GzipFile(mode="wb",
compresslevel=COMPRESS_LEVEL,
fileobj=f)
gz.write(data)
gz.close()
self.data = f.getvalue()
f.close()
elif self.encoding == self.ENCODE_ZLIB:
import zlib
encoding_name = self.encodings[self.ENCODE_ZLIB][0]
self.set_header("Content-Encoding", encoding_name)
obj = zlib.compressobj(COMPRESS_LEVEL)
self.data = obj.compress(data) + obj.flush()
elif self.encoding == self.ENCODE_GPG:
# XXX: fix me.
raise NotImplementedError(self.transfer, self.encoding)
encoding_name = self.encodings[self.ENCODE_GPG][0]
self.set_header("Content-Encoding", encoding_name)
# Content-Transfer-Encoding header
if self.transfer == self.TRANSFER_BINARY:
transfer_name = self.transfers[self.TRANSFER_BINARY]
self.set_header("Content-Transfer-Encoding", transfer_name)
self.set_header("Content-Type", "application/binary")
elif self.transfer == self.TRANSFER_BASE64:
import base64
transfer_name = self.transfers[self.TRANSFER_BASE64]
self.set_header("Content-Transfer-Encoding", transfer_name)
self.set_header("Content-Type", "text/base64")
self.data = base64.encodestring(self.data)
self.set_header("Content-Length", len(self.data))
rpc_version = __version__
if len(__version__.split()) > 1:
rpc_version = __version__.split()[1]
# other headers
self.set_header(
"X-Transport-Info",
'Extended Capabilities Transport (C) Red Hat, Inc (version %s)' %
rpc_version)
self.__processed = 1
def process(self, data):
# Assume straight text/xml
self.data = data
# Content-Encoding header
if self.encoding == self.ENCODE_GZIP:
import gzip
encoding_name = self.encodings[self.ENCODE_GZIP][0]
self.set_header("Content-Encoding", encoding_name)
f = SmartIO(force_mem=1)
gz = gzip.GzipFile(mode="wb", compresslevel=COMPRESS_LEVEL,
fileobj = f)
gz.write(data)
gz.close()
self.data = f.getvalue()
f.close()
elif self.encoding == self.ENCODE_ZLIB:
import zlib
encoding_name = self.encodings[self.ENCODE_ZLIB][0]
self.set_header("Content-Encoding", encoding_name)
obj = zlib.compressobj(COMPRESS_LEVEL)
self.data = obj.compress(data) + obj.flush()
elif self.encoding == self.ENCODE_GPG:
# XXX: fix me.
raise NotImplementedError(self.transfer, self.encoding)
encoding_name = self.encodings[self.ENCODE_GPG][0]
self.set_header("Content-Encoding", encoding_name)
# Content-Transfer-Encoding header
if self.transfer == self.TRANSFER_BINARY:
transfer_name = self.transfers[self.TRANSFER_BINARY]
self.set_header("Content-Transfer-Encoding", transfer_name)
self.set_header("Content-Type", "application/binary")
elif self.transfer == self.TRANSFER_BASE64:
import base64
transfer_name = self.transfers[self.TRANSFER_BASE64]
self.set_header("Content-Transfer-Encoding", transfer_name)
self.set_header("Content-Type", "text/base64")
self.data = base64.encodestring(self.data)
self.set_header("Content-Length", len(self.data))
rpc_version = __version__
if len(__version__.split()) > 1:
rpc_version = __version__.split()[1]
# other headers
self.set_header("X-Transport-Info",
'Extended Capabilities Transport (C) Red Hat, Inc (version %s)' %
rpc_version)
self.__processed = 1
def decode(self, fd=sys.stdin):
# The octet-stream data are passed right back
if self.type == "application/octet-stream":
return InputStream(fd, self.length, self.name, close=fd.close)
if not self.io:
self.read(fd)
# At this point self.io exists (the only case when self.read() does
# not initialize self.io is when content-type is
# "application/octet-stream" - and we already dealt with that case
# We can now close the file descriptor
if hasattr(fd, "close"):
fd.close()
# Now we have the binary goo
if not self.encoding or self.encoding == "__plain":
# all is fine.
pass
elif self.encoding in ("x-zlib", "deflate"):
import zlib
obj = zlib.decompressobj()
self.io.seek(0, 0)
data = obj.decompress(self.io.read()) + obj.flush()
del obj
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding in ("x-gzip", "gzip"):
import gzip
self.io.seek(0, 0)
gz = gzip.GzipFile(mode="rb",
compresslevel=COMPRESS_LEVEL,
fileobj=self.io)
data = gz.read()
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding == "x-gpg":
# XXX: should be written
raise NotImplementedError(self.transfer, self.encoding)
else:
raise NotImplementedError(self.transfer, self.encoding)
# Play nicely and rewind the file descriptor
self.io.seek(0, 0)
return self.io
def _smart_total_read(fd, bufferSize=1024, max_mem_size=16384):
"""
Tries to read data from the supplied stream, and puts the results into a
StmartIO object. The data will be in memory or in a temporary file,
depending on how much it's been read
Returns a SmartIO object
"""
io = SmartIO(max_mem_size=max_mem_size)
while 1:
chunk = fd.read(bufferSize)
if not chunk:
# EOF reached
break
io.write(chunk)
return io
def read(self, fd = sys.stdin):
# The octet-streams are passed right back
if self.type == "application/octet-stream":
return
if self.length:
# Read exactly the amount of data we were told
self.io = _smart_read(fd, self.length,
bufferSize=self.bufferSize,
progressCallback=self.progressCallback,
max_mem_size=self.max_mem_size)
else:
# Oh well, no clue; read until EOF (hopefully)
self.io = _smart_total_read(fd)
if not self.transfer or self.transfer == "binary":
return
elif self.transfer == "base64":
import base64
old_io = self.io
old_io.seek(0, 0)
self.io = SmartIO(max_mem_size=self.max_mem_size)
base64.decode(old_io, self.io)
else:
raise NotImplementedError(self.transfer)
def _smart_total_read(fd, bufferSize=1024, max_mem_size=16384):
"""
Tries to read data from the supplied stream, and puts the results into a
StmartIO object. The data will be in memory or in a temporary file,
depending on how much it's been read
Returns a SmartIO object
"""
io = SmartIO(max_mem_size=max_mem_size)
while 1:
chunk = fd.read(bufferSize)
if not chunk:
# EOF reached
break
io.write(chunk)
return io
def decode(self, fd = sys.stdin):
# The octet-stream data are passed right back
if self.type == "application/octet-stream":
return InputStream(fd, self.length, self.name, close=fd.close)
if not self.io:
self.read(fd)
# At this point self.io exists (the only case when self.read() does
# not initialize self.io is when content-type is
# "application/octet-stream" - and we already dealt with that case
# We can now close the file descriptor
if hasattr(fd, "close"):
fd.close()
# Now we have the binary goo
if not self.encoding or self.encoding == "__plain":
# all is fine.
pass
elif self.encoding in ("x-zlib", "deflate"):
import zlib
obj = zlib.decompressobj()
self.io.seek(0, 0)
data = obj.decompress(self.io.read()) + obj.flush()
del obj
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding in ("x-gzip", "gzip"):
import gzip
self.io.seek(0, 0)
gz = gzip.GzipFile(mode="rb", compresslevel = COMPRESS_LEVEL,
fileobj=self.io)
data = gz.read()
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding == "x-gpg":
# XXX: should be written
raise NotImplementedError(self.transfer, self.encoding)
else:
raise NotImplementedError(self.transfer, self.encoding)
# Play nicely and rewind the file descriptor
self.io.seek(0, 0)
return self.io
def _smart_read(fd, amt, bufferSize=1024, progressCallback=None,
max_mem_size=16384):
# Reads amt bytes from fd, or until the end of file, whichever
# occurs first
# The function will read in memory if the amout to be read is smaller than
# max_mem_size, or to a temporary file otherwise
#
# Unlike read(), _smart_read tries to return exactly the requested amount
# (whereas read will return _up_to_ that amount). Reads from sockets will
# usually reaturn less data, or the read can be interrupted
#
# Inspired by Greg Stein's httplib.py (the standard in python 2.x)
#
# support for progress callbacks added
startTime = time.time()
lastTime = startTime
buf = SmartIO(max_mem_size=max_mem_size)
origsize = amt
while amt > 0:
curTime = time.time()
l = min(bufferSize, amt)
chunk = fd.read(l)
# read guarantees that len(chunk) <= l
l = len(chunk)
if not l:
# Oops. Most likely EOF
break
# And since the original l was smaller than amt, we know amt >= 0
amt = amt - l
buf.write(chunk)
if progressCallback is None:
# No progress callback, so don't do fancy computations
continue
# We update the progress callback if:
# we haven't updated it for more than a secord, or
# it's the last read (amt == 0)
if curTime - lastTime >= 1 or amt == 0:
lastTime = curTime
# use float() so that we force float division in the next step
bytesRead = float(origsize - amt)
# if amt == 0, on a fast machine it is possible to have
# curTime - lastTime == 0, so add an epsilon to prevent a division
# by zero
speed = bytesRead / ((curTime - startTime) + .000001)
if origsize == 0:
secs = 0
else:
# speed != 0 because bytesRead > 0
# (if bytesRead == 0 then origsize == amt, which means a read
# of 0 length; but that's impossible since we already checked
# that l is non-null
secs = amt / speed
progressCallback(bytesRead, origsize, speed, secs)
# Now rewind the SmartIO
buf.seek(0, 0)
return buf
class Input:
def __init__(self, headers=None, progressCallback=None, bufferSize=1024,
max_mem_size=16384):
self.transfer = None
self.encoding = None
self.type = None
self.length = 0
self.lang = "C"
self.name = ""
self.progressCallback = progressCallback
self.bufferSize = bufferSize
self.max_mem_size = max_mem_size
if not headers:
# we need to get them from environment
if os.environ.has_key("HTTP_CONTENT_TRANSFER_ENCODING"):
self.transfer = os.environ["HTTP_CONTENT_TRANSFER_ENCODING"].lower()
if os.environ.has_key("HTTP_CONTENT_ENCODING"):
self.encoding = os.environ["HTTP_CONTENT_ENCODING"].lower()
if os.environ.has_key("CONTENT-TYPE"):
self.type = os.environ["CONTENT-TYPE"].lower()
if os.environ.has_key("CONTENT_LENGTH"):
self.length = int(os.environ["CONTENT_LENGTH"])
if os.environ.has_key("HTTP_ACCEPT_LANGUAGE"):
self.lang = os.environ["HTTP_ACCEPT_LANGUAGE"]
if os.environ.has_key("HTTP_X_PACKAGE_FILENAME"):
self.name = os.environ["HTTP_X_PACKAGE_FILENAME"]
else:
# The stupid httplib screws up the headers from the HTTP repsonse
# and converts them to lowercase. This means that we have to
# convert to lowercase all the dictionary keys in case somebody calls
# us with sane values --gaftonc (actually mimetools is the culprit)
for header in headers.keys():
value = headers[header]
h = header.lower()
if h == "content-length":
try:
self.length = int(value)
except ValueError:
self.length = 0
elif h == "content-transfer-encoding":
# RFC 2045 #6.1: case insensitive
self.transfer = value.lower()
elif h == "content-encoding":
# RFC 2616 #3.5: case insensitive
self.encoding = value.lower()
elif h == "content-type":
# RFC 2616 #3.7: case insensitive
self.type = value.lower()
elif h == "accept-language":
# RFC 2616 #3.10: case insensitive
self.lang = value.lower()
elif h == "x-package-filename":
self.name = value
self.io = None
def read(self, fd = sys.stdin):
# The octet-streams are passed right back
if self.type == "application/octet-stream":
return
if self.length:
# Read exactly the amount of data we were told
self.io = _smart_read(fd, self.length,
bufferSize=self.bufferSize,
progressCallback=self.progressCallback,
max_mem_size=self.max_mem_size)
else:
# Oh well, no clue; read until EOF (hopefully)
self.io = _smart_total_read(fd)
if not self.transfer or self.transfer == "binary":
return
elif self.transfer == "base64":
import base64
old_io = self.io
old_io.seek(0, 0)
self.io = SmartIO(max_mem_size=self.max_mem_size)
base64.decode(old_io, self.io)
else:
raise NotImplementedError(self.transfer)
def decode(self, fd = sys.stdin):
# The octet-stream data are passed right back
if self.type == "application/octet-stream":
return InputStream(fd, self.length, self.name, close=fd.close)
if not self.io:
self.read(fd)
# At this point self.io exists (the only case when self.read() does
# not initialize self.io is when content-type is
# "application/octet-stream" - and we already dealt with that case
# We can now close the file descriptor
if hasattr(fd, "close"):
fd.close()
# Now we have the binary goo
if not self.encoding or self.encoding == "__plain":
# all is fine.
pass
elif self.encoding in ("x-zlib", "deflate"):
import zlib
obj = zlib.decompressobj()
self.io.seek(0, 0)
data = obj.decompress(self.io.read()) + obj.flush()
del obj
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding in ("x-gzip", "gzip"):
import gzip
self.io.seek(0, 0)
gz = gzip.GzipFile(mode="rb", compresslevel = COMPRESS_LEVEL,
fileobj=self.io)
data = gz.read()
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding == "x-gpg":
# XXX: should be written
raise NotImplementedError(self.transfer, self.encoding)
else:
raise NotImplementedError(self.transfer, self.encoding)
# Play nicely and rewind the file descriptor
self.io.seek(0, 0)
return self.io
def getlang(self):
return self.lang
class Input:
def __init__(self,
headers=None,
progressCallback=None,
bufferSize=1024,
max_mem_size=16384):
self.transfer = None
self.encoding = None
self.type = None
self.length = 0
self.lang = "C"
self.name = ""
self.progressCallback = progressCallback
self.bufferSize = bufferSize
self.max_mem_size = max_mem_size
if not headers:
# we need to get them from environment
if os.environ.has_key("HTTP_CONTENT_TRANSFER_ENCODING"):
self.transfer = os.environ[
"HTTP_CONTENT_TRANSFER_ENCODING"].lower()
if os.environ.has_key("HTTP_CONTENT_ENCODING"):
self.encoding = os.environ["HTTP_CONTENT_ENCODING"].lower()
if os.environ.has_key("CONTENT-TYPE"):
self.type = os.environ["CONTENT-TYPE"].lower()
if os.environ.has_key("CONTENT_LENGTH"):
self.length = int(os.environ["CONTENT_LENGTH"])
if os.environ.has_key("HTTP_ACCEPT_LANGUAGE"):
self.lang = os.environ["HTTP_ACCEPT_LANGUAGE"]
if os.environ.has_key("HTTP_X_PACKAGE_FILENAME"):
self.name = os.environ["HTTP_X_PACKAGE_FILENAME"]
else:
# The stupid httplib screws up the headers from the HTTP repsonse
# and converts them to lowercase. This means that we have to
# convert to lowercase all the dictionary keys in case somebody calls
# us with sane values --gaftonc (actually mimetools is the culprit)
for header in headers.keys():
value = headers[header]
h = header.lower()
if h == "content-length":
try:
self.length = int(value)
except ValueError:
self.length = 0
elif h == "content-transfer-encoding":
# RFC 2045 #6.1: case insensitive
self.transfer = value.lower()
elif h == "content-encoding":
# RFC 2616 #3.5: case insensitive
self.encoding = value.lower()
elif h == "content-type":
# RFC 2616 #3.7: case insensitive
self.type = value.lower()
elif h == "accept-language":
# RFC 2616 #3.10: case insensitive
self.lang = value.lower()
elif h == "x-package-filename":
self.name = value
self.io = None
def read(self, fd=sys.stdin):
# The octet-streams are passed right back
if self.type == "application/octet-stream":
return
if self.length:
# Read exactly the amount of data we were told
self.io = _smart_read(fd,
self.length,
bufferSize=self.bufferSize,
progressCallback=self.progressCallback,
max_mem_size=self.max_mem_size)
else:
# Oh well, no clue; read until EOF (hopefully)
self.io = _smart_total_read(fd)
if not self.transfer or self.transfer == "binary":
return
elif self.transfer == "base64":
import base64
old_io = self.io
old_io.seek(0, 0)
self.io = SmartIO(max_mem_size=self.max_mem_size)
base64.decode(old_io, self.io)
else:
raise NotImplementedError(self.transfer)
def decode(self, fd=sys.stdin):
# The octet-stream data are passed right back
if self.type == "application/octet-stream":
return InputStream(fd, self.length, self.name, close=fd.close)
if not self.io:
self.read(fd)
# At this point self.io exists (the only case when self.read() does
# not initialize self.io is when content-type is
# "application/octet-stream" - and we already dealt with that case
# We can now close the file descriptor
if hasattr(fd, "close"):
fd.close()
# Now we have the binary goo
if not self.encoding or self.encoding == "__plain":
# all is fine.
pass
elif self.encoding in ("x-zlib", "deflate"):
import zlib
obj = zlib.decompressobj()
self.io.seek(0, 0)
data = obj.decompress(self.io.read()) + obj.flush()
del obj
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding in ("x-gzip", "gzip"):
import gzip
self.io.seek(0, 0)
gz = gzip.GzipFile(mode="rb",
compresslevel=COMPRESS_LEVEL,
fileobj=self.io)
data = gz.read()
self.length = len(data)
self.io = SmartIO(max_mem_size=self.max_mem_size)
self.io.write(data)
elif self.encoding == "x-gpg":
# XXX: should be written
raise NotImplementedError(self.transfer, self.encoding)
else:
raise NotImplementedError(self.transfer, self.encoding)
# Play nicely and rewind the file descriptor
self.io.seek(0, 0)
return self.io
def getlang(self):
return self.lang
