def test_compressed_wrapper(self):
r = compression.compressed_wrapper("test", b"a"*(compression.MIN_COMPRESS_SIZE+1))
if not r.datatype.startswith("raw"):
raise Exception("should not be able to use the wrapper without enabling a compressor, but got %s" % r)
for x in ("lz4", "brotli", "zlib", "none"):
if not compression.use(x):
continue
kwargs = {x : True}
for level in (0, 1, 5, 10):
for data in (
b"0"*1024,
b"0"*16,
b"\0",
memoryview(b"hello"),
bytearray(b"hello"),
b"1"*1024*1024*16,
):
v = compression.compressed_wrapper("test", data, level=level, **kwargs)
assert v
assert repr(v)
assert compression.get_compression_type(v.level)
#and back:
try:
d = compression.decompress_by_name(v.data, v.algorithm)
assert d
if x!="none":
#we can't do none,
#because it would be mistaken for "zlib"
#(for historical reasons - 'zlib' uses level=0, and 'none' has no level)
d = compression.decompress(v.data, v.level)
assert d
except Exception:
print("error decompressing %s - generated with settings: %s" % (v, kwargs))
raise
def do_read_parse_thread_loop(self):
"""
Process the individual network packets placed in _read_queue.
Concatenate the raw packet data, then try to parse it.
Extract the individual packets from the potentially large buffer,
saving the rest of the buffer for later, and optionally decompress this data
and re-construct the one python-object-packet from potentially multiple packets (see packet_index).
The 8 bytes packet header gives us information on the packet index, packet size and compression.
The actual processing of the packet is done via the callback process_packet_cb,
this will be called from this parsing thread so any calls that need to be made
from the UI thread will need to use a callback (usually via 'idle_add')
"""
read_buffer = None
payload_size = -1
padding_size = 0
packet_index = 0
compression_level = False
packet = None
raw_packets = {}
while not self._closed:
buf = self._read_queue.get()
if not buf:
log("parse thread: empty marker, exiting")
self.idle_add(self.close)
return
if read_buffer:
read_buffer = read_buffer + buf
else:
read_buffer = buf
bl = len(read_buffer)
while not self._closed:
packet = None
bl = len(read_buffer)
if bl <= 0:
break
if payload_size < 0:
if read_buffer[0] not in ("P", ord("P")):
self._invalid_header(read_buffer)
return
if bl < 8:
break #packet still too small
#packet format: struct.pack('cBBBL', ...) - 8 bytes
_, protocol_flags, compression_level, packet_index, data_size = unpack_header(
read_buffer[:8])
#sanity check size (will often fail if not an xpra client):
if data_size > self.abs_max_packet_size:
self._invalid_header(read_buffer)
return
bl = len(read_buffer) - 8
if protocol_flags & FLAGS_CIPHER:
if self.cipher_in_block_size == 0 or not self.cipher_in_name:
cryptolog.warn(
"received cipher block but we don't have a cipher to decrypt it with, not an xpra client?"
)
self._invalid_header(read_buffer)
return
padding_size = self.cipher_in_block_size - (
data_size % self.cipher_in_block_size)
payload_size = data_size + padding_size
else:
#no cipher, no padding:
padding_size = 0
payload_size = data_size
assert payload_size > 0, "invalid payload size: %i" % payload_size
read_buffer = read_buffer[8:]
if payload_size > self.max_packet_size:
#this packet is seemingly too big, but check again from the main UI thread
#this gives 'set_max_packet_size' a chance to run from "hello"
def check_packet_size(size_to_check, packet_header):
if self._closed:
return False
log("check_packet_size(%s, 0x%s) limit is %s",
size_to_check, repr_ellipsized(packet_header),
self.max_packet_size)
if size_to_check > self.max_packet_size:
msg = "packet size requested is %s but maximum allowed is %s" % \
(size_to_check, self.max_packet_size)
self.invalid(msg, packet_header)
return False
self.timeout_add(1000, check_packet_size, payload_size,
read_buffer[:32])
if bl < payload_size:
# incomplete packet, wait for the rest to arrive
break
#chop this packet from the buffer:
if len(read_buffer) == payload_size:
raw_string = read_buffer
read_buffer = ''
else:
raw_string = read_buffer[:payload_size]
read_buffer = read_buffer[payload_size:]
#decrypt if needed:
data = raw_string
if self.cipher_in and protocol_flags & FLAGS_CIPHER:
cryptolog("received %i %s encrypted bytes with %s padding",
payload_size, self.cipher_in_name, padding_size)
data = self.cipher_in.decrypt(raw_string)
if padding_size > 0:
def debug_str(s):
try:
return binascii.hexlify(bytearray(s))
except:
return csv(list(str(s)))
# pad byte value is number of padding bytes added
padtext = pad(self.cipher_in_padding, padding_size)
if data.endswith(padtext):
cryptolog("found %s %s padding",
self.cipher_in_padding,
self.cipher_in_name)
else:
actual_padding = data[-padding_size:]
cryptolog.warn(
"Warning: %s decryption failed: invalid padding",
self.cipher_in_name)
cryptolog(
" data does not end with %s padding bytes %s",
self.cipher_in_padding, debug_str(padtext))
cryptolog(" but with %s (%s)",
debug_str(actual_padding), type(data))
cryptolog(" decrypted data: %s",
debug_str(data[:128]))
return self._internal_error(
"%s encryption padding error - wrong key?" %
self.cipher_in_name)
data = data[:-padding_size]
#uncompress if needed:
if compression_level > 0:
try:
data = decompress(data, compression_level)
except InvalidCompressionException as e:
self.invalid("invalid compression: %s" % e, data)
return
except Exception as e:
ctype = compression.get_compression_type(
compression_level)
log("%s packet decompression failed",
ctype,
exc_info=True)
msg = "%s packet decompression failed" % ctype
if self.cipher_in:
msg += " (invalid encryption key?)"
else:
#only include the exception text when not using encryption
#as this may leak crypto information:
msg += " %s" % e
return self.gibberish(msg, data)
if self.cipher_in and not (protocol_flags & FLAGS_CIPHER):
self.invalid("unencrypted packet dropped", data)
return
if self._closed:
return
if packet_index > 0:
#raw packet, store it and continue:
raw_packets[packet_index] = data
payload_size = -1
packet_index = 0
if len(raw_packets) >= 4:
self.invalid(
"too many raw packets: %s" % len(raw_packets),
data)
return
continue
#final packet (packet_index==0), decode it:
try:
packet = decode(data, protocol_flags)
except InvalidPacketEncodingException as e:
self.invalid("invalid packet encoding: %s" % e, data)
return
except ValueError as e:
etype = packet_encoding.get_packet_encoding_type(
protocol_flags)
log.error("Error parsing %s packet:", etype)
log.error(" %s", e)
if self._closed:
return
log("failed to parse %s packet: %s", etype,
binascii.hexlify(data[:128]))
log(" %s", e)
log(" data: %s", repr_ellipsized(data))
log(" packet index=%i, packet size=%i, buffer size=%s",
packet_index, payload_size, bl)
self.gibberish("failed to parse %s packet" % etype, data)
return
if self._closed:
return
payload_size = -1
padding_size = 0
#add any raw packets back into it:
if raw_packets:
for index, raw_data in raw_packets.items():
#replace placeholder with the raw_data packet data:
packet[index] = raw_data
raw_packets = {}
packet_type = packet[0]
if self.receive_aliases and type(
packet_type
) == int and packet_type in self.receive_aliases:
packet_type = self.receive_aliases.get(packet_type)
packet[0] = packet_type
self.input_stats[packet_type] = self.output_stats.get(
packet_type, 0) + 1
self.input_packetcount += 1
log("processing packet %s", packet_type)
self._process_packet_cb(self, packet)
packet = None
INJECT_FAULT(self)
def do_read_parse_thread_loop(self):
"""
Process the individual network packets placed in _read_queue.
Concatenate the raw packet data, then try to parse it.
Extract the individual packets from the potentially large buffer,
saving the rest of the buffer for later, and optionally decompress this data
and re-construct the one python-object-packet from potentially multiple packets (see packet_index).
The 8 bytes packet header gives us information on the packet index, packet size and compression.
The actual processing of the packet is done via the callback process_packet_cb,
this will be called from this parsing thread so any calls that need to be made
from the UI thread will need to use a callback (usually via 'idle_add')
"""
read_buffer = None
payload_size = -1
padding = None
packet_index = 0
compression_level = False
packet = None
raw_packets = {}
while not self._closed:
buf = self._read_queue.get()
if not buf:
log("read thread: empty marker, exiting")
self.idle_add(self.close)
return
if read_buffer:
read_buffer = read_buffer + buf
else:
read_buffer = buf
bl = len(read_buffer)
while not self._closed:
packet = None
bl = len(read_buffer)
if bl <= 0:
break
if payload_size < 0:
head = read_buffer[:8]
if read_buffer[0] not in ("P", ord("P")):
self._invalid_header(read_buffer)
return
if bl < 8:
break #packet still too small
#packet format: struct.pack('cBBBL', ...) - 8 bytes
_, protocol_flags, compression_level, packet_index, data_size = unpack_header(
head)
#sanity check size (will often fail if not an xpra client):
if data_size > self.abs_max_packet_size:
self._invalid_header(read_buffer)
return
bl = len(read_buffer) - 8
if protocol_flags & FLAGS_CIPHER:
if self.cipher_in_block_size == 0 or not self.cipher_in_name:
log.warn(
"received cipher block but we don't have a cipher to decrypt it with, not an xpra client?"
)
self._invalid_header(read_buffer)
return
padding = (self.cipher_in_block_size -
data_size % self.cipher_in_block_size) * " "
payload_size = data_size + len(padding)
else:
#no cipher, no padding:
padding = None
payload_size = data_size
assert payload_size > 0
read_buffer = read_buffer[8:]
if payload_size > self.max_packet_size:
#this packet is seemingly too big, but check again from the main UI thread
#this gives 'set_max_packet_size' a chance to run from "hello"
def check_packet_size(size_to_check, packet_header):
if self._closed:
return False
log("check_packet_size(%s, 0x%s) limit is %s",
size_to_check, repr_ellipsized(packet_header),
self.max_packet_size)
if size_to_check > self.max_packet_size:
msg = "packet size requested is %s but maximum allowed is %s" % \
(size_to_check, self.max_packet_size)
self.invalid(msg, packet_header)
return False
self.timeout_add(1000, check_packet_size, payload_size,
read_buffer[:32])
if bl < payload_size:
# incomplete packet, wait for the rest to arrive
break
#chop this packet from the buffer:
if len(read_buffer) == payload_size:
raw_string = read_buffer
read_buffer = ''
else:
raw_string = read_buffer[:payload_size]
read_buffer = read_buffer[payload_size:]
#decrypt if needed:
data = raw_string
if self.cipher_in and protocol_flags & FLAGS_CIPHER:
log("received %s encrypted bytes with %s padding",
payload_size, len(padding))
data = self.cipher_in.decrypt(raw_string)
if padding:
def debug_str(s):
try:
return list(bytearray(s))
except:
return list(str(s))
if not data.endswith(padding):
log(
"decryption failed: string does not end with '%s': %s (%s) -> %s (%s)",
padding, debug_str(raw_string),
type(raw_string), debug_str(data), type(data))
self._internal_error(
"encryption error (wrong key?)")
return
data = data[:-len(padding)]
#uncompress if needed:
if compression_level > 0:
try:
data = decompress(data, compression_level)
except InvalidCompressionException, e:
self.invalid("invalid compression: %s" % e, data)
return
except Exception, e:
ctype = compression.get_compression_type(
compression_level)
log("%s packet decompression failed",
ctype,
exc_info=True)
msg = "%s packet decompression failed" % ctype
if self.cipher_in:
msg += " (invalid encryption key?)"
else:
msg += " %s" % e
return self.gibberish(msg, data)
if self.cipher_in and not (protocol_flags & FLAGS_CIPHER):
self.invalid("unencrypted packet dropped", data)
return
if self._closed:
return
if packet_index > 0:
#raw packet, store it and continue:
raw_packets[packet_index] = data
payload_size = -1
packet_index = 0
if len(raw_packets) >= 4:
self.invalid(
"too many raw packets: %s" % len(raw_packets),
data)
return
continue
#final packet (packet_index==0), decode it:
try:
packet = decode(data, protocol_flags)
except InvalidPacketEncodingException, e:
self.invalid("invalid packet encoding: %s" % e, data)
return
def do_read_parse_thread_loop(self):
"""
Process the individual network packets placed in _read_queue.
Concatenate the raw packet data, then try to parse it.
Extract the individual packets from the potentially large buffer,
saving the rest of the buffer for later, and optionally decompress this data
and re-construct the one python-object-packet from potentially multiple packets (see packet_index).
The 8 bytes packet header gives us information on the packet index, packet size and compression.
The actual processing of the packet is done via the callback process_packet_cb,
this will be called from this parsing thread so any calls that need to be made
from the UI thread will need to use a callback (usually via 'idle_add')
"""
read_buffer = None
payload_size = -1
padding = None
packet_index = 0
compression_level = False
packet = None
raw_packets = {}
while not self._closed:
buf = self._read_queue.get()
if not buf:
log("read thread: empty marker, exiting")
self.idle_add(self.close)
return
if read_buffer:
read_buffer = read_buffer + buf
else:
read_buffer = buf
bl = len(read_buffer)
while not self._closed:
packet = None
bl = len(read_buffer)
if bl <= 0:
break
if payload_size < 0:
if read_buffer[0] not in ("P", ord("P")):
self._invalid_header(read_buffer)
return
if bl < 8:
break # packet still too small
# packet format: struct.pack('cBBBL', ...) - 8 bytes
_, protocol_flags, compression_level, packet_index, data_size = unpack_header(read_buffer[:8])
# sanity check size (will often fail if not an xpra client):
if data_size > self.abs_max_packet_size:
self._invalid_header(read_buffer)
return
bl = len(read_buffer) - 8
if protocol_flags & FLAGS_CIPHER:
if self.cipher_in_block_size == 0 or not self.cipher_in_name:
log.warn(
"received cipher block but we don't have a cipher to decrypt it with, not an xpra client?"
)
self._invalid_header(read_buffer)
return
padding = (self.cipher_in_block_size - data_size % self.cipher_in_block_size) * " "
payload_size = data_size + len(padding)
else:
# no cipher, no padding:
padding = None
payload_size = data_size
assert payload_size > 0
read_buffer = read_buffer[8:]
if payload_size > self.max_packet_size:
# this packet is seemingly too big, but check again from the main UI thread
# this gives 'set_max_packet_size' a chance to run from "hello"
def check_packet_size(size_to_check, packet_header):
if self._closed:
return False
log(
"check_packet_size(%s, 0x%s) limit is %s",
size_to_check,
repr_ellipsized(packet_header),
self.max_packet_size,
)
if size_to_check > self.max_packet_size:
msg = "packet size requested is %s but maximum allowed is %s" % (
size_to_check,
self.max_packet_size,
)
self.invalid(msg, packet_header)
return False
self.timeout_add(1000, check_packet_size, payload_size, read_buffer[:32])
if bl < payload_size:
# incomplete packet, wait for the rest to arrive
break
# chop this packet from the buffer:
if len(read_buffer) == payload_size:
raw_string = read_buffer
read_buffer = ""
else:
raw_string = read_buffer[:payload_size]
read_buffer = read_buffer[payload_size:]
# decrypt if needed:
data = raw_string
if self.cipher_in and protocol_flags & FLAGS_CIPHER:
log("received %s encrypted bytes with %s padding", payload_size, len(padding))
data = self.cipher_in.decrypt(raw_string)
if padding:
def debug_str(s):
try:
return list(bytearray(s))
except:
return list(str(s))
if not data.endswith(padding):
log(
"decryption failed: string does not end with '%s': %s (%s) -> %s (%s)",
padding,
debug_str(raw_string),
type(raw_string),
debug_str(data),
type(data),
)
self._internal_error("encryption error (wrong key?)")
return
data = data[: -len(padding)]
# uncompress if needed:
if compression_level > 0:
try:
data = decompress(data, compression_level)
except InvalidCompressionException as e:
self.invalid("invalid compression: %s" % e, data)
return
except Exception as e:
ctype = compression.get_compression_type(compression_level)
log("%s packet decompression failed", ctype, exc_info=True)
msg = "%s packet decompression failed" % ctype
if self.cipher_in:
msg += " (invalid encryption key?)"
else:
msg += " %s" % e
return self.gibberish(msg, data)
if self.cipher_in and not (protocol_flags & FLAGS_CIPHER):
self.invalid("unencrypted packet dropped", data)
return
if self._closed:
return
if packet_index > 0:
# raw packet, store it and continue:
raw_packets[packet_index] = data
payload_size = -1
packet_index = 0
if len(raw_packets) >= 4:
self.invalid("too many raw packets: %s" % len(raw_packets), data)
return
continue
# final packet (packet_index==0), decode it:
try:
packet = decode(data, protocol_flags)
except InvalidPacketEncodingException as e:
self.invalid("invalid packet encoding: %s" % e, data)
return
except ValueError as e:
etype = packet_encoding.get_packet_encoding_type(protocol_flags)
log.error("failed to parse %s packet: %s", etype, e, exc_info=not self._closed)
if self._closed:
return
log("failed to parse %s packet: %s", etype, binascii.hexlify(data))
msg = "packet index=%s, packet size=%s, buffer size=%s, error=%s" % (
packet_index,
payload_size,
bl,
e,
)
self.gibberish("failed to parse %s packet" % etype, data)
return
if self._closed:
return
payload_size = -1
padding = None
# add any raw packets back into it:
if raw_packets:
for index, raw_data in raw_packets.items():
# replace placeholder with the raw_data packet data:
packet[index] = raw_data
raw_packets = {}
packet_type = packet[0]
if self.receive_aliases and type(packet_type) == int and packet_type in self.receive_aliases:
packet_type = self.receive_aliases.get(packet_type)
packet[0] = packet_type
self.input_stats[packet_type] = self.output_stats.get(packet_type, 0) + 1
self.input_packetcount += 1
log("processing packet %s", packet_type)
self._process_packet_cb(self, packet)
packet = None