def parse_raw_packet(self, packet: bytes, crc_len_format="L", number_of_chunks_len_format="I",
quality_len_format="I", epsilon_len_format="f", check_block_number_len_format="I") -> \
typing.Optional[typing.Union[str, OnlinePacket]]:
crc_len = -struct.calcsize("<" + crc_len_format)
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
payload = packet[:crc_len]
crc: int = struct.unpack("<" + crc_len_format, packet[crc_len:])[0]
calced_crc: int = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " +
str(hex(calced_crc)))
self.corrupt += 1
return "CORRUPT"
else:
crc_len = None
try:
packet = self.error_correction(packet)
except:
return "CORRUPT" # if RS or other error correction cannot reconstruct this packet
struct_str: str = "<" + number_of_chunks_len_format + quality_len_format + epsilon_len_format + check_block_number_len_format
struct_len: int = struct.calcsize(struct_str)
data: bytes = packet[struct_len:crc_len]
len_data: typing.Union[typing.Tuple[int, int, float, int],
typing.Tuple[int, float, int]] = struct.unpack(
struct_str, packet[0:struct_len])
if self.static_number_of_chunks is None:
number_of_chunks, quality, self.epsilon, check_block_number = len_data
self.number_of_chunks = xor_mask(number_of_chunks,
number_of_chunks_len_format)
else:
quality, self.epsilon, check_block_number = len_data
self.epsilon = round(self.epsilon, 6)
self.quality = xor_mask(quality, quality_len_format)
if self.dist is None:
self.dist = OnlineDistribution(self.epsilon)
if self.correct == 0:
self.createAuxBlocks()
# Create MockUp AuxBlocks with the given Pseudo-Random Number -> we will know which Packets are Encoded in which AuxBlock
self.correct += 1
res = OnlinePacket(
data,
self.number_of_chunks,
self.quality,
self.epsilon,
check_block_number,
dist=self.dist,
read_only=True,
error_correction=self.error_correction,
crc_len_format=crc_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
quality_len_format=quality_len_format,
epsilon_len_format=epsilon_len_format,
check_block_number_len_format=check_block_number_len_format,
save_number_of_chunks_in_packet=self.static_number_of_chunks is
None)
return res
def prepare_and_pack(self) -> bytes:
# Format = Highest possible Packetnumber for this file,
# number of used Packets for this File and the seed for the Indices of the used Packets
if self.save_number_of_chunks_in_packet:
return struct.pack(
"<" + self.number_of_chunks_len_format + self.id_len_format,
xor_mask(self.total_number_of_chunks,
self.number_of_chunks_len_format),
xor_mask(self.id, self.id_len_format))
else:
return struct.pack("<" + self.id_len_format,
xor_mask(self.id, self.id_len_format))
def listen(sock, queue, signals):
if sock is None:
return None
while not signals["shutdown"]:
try:
data, sender = sock.recvfrom(1024)
a, b = struct.unpack("<II", data[0:8])
print("Packet from: %s:%s to %s - #Chunks: %s - Id: %s" %
(sender[0], sender[1], sock.getsockname()[0], xor_mask(a),
xor_mask(b)))
queue.put(data)
except socket.error as ex:
# queue.put(e)
continue
def parse_raw_packet(self, packet: bytes, crc_len_format: str = "L", number_of_chunks_len_format: str = "I",
degree_len_format: str = "I", seed_len_format: str = "I") -> typing.Union[str, Packet]:
crc_len = -struct.calcsize("<" + crc_len_format)
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
payload: bytes = packet[:crc_len]
crc: int = struct.unpack("<" + crc_len_format, packet[crc_len:])[0]
calced_crc: int = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc)))
self.corrupt += 1
return "CORRUPT"
else:
crc_len = None
try:
packet = self.error_correction(packet)
except:
self.corrupt += 1
return "CORRUPT"
if self.implicit_mode:
degree_len_format = ""
struct_str: str = "<" + number_of_chunks_len_format + degree_len_format + seed_len_format
struct_len: int = struct.calcsize(struct_str)
len_data: typing.Union[int, typing.Tuple[int, int], typing.Tuple[int, int, int]] = struct.unpack(struct_str,
packet[
0:struct_len])
degree: typing.Optional[int] = None
if self.static_number_of_chunks is None:
if self.implicit_mode:
number_of_chunks, seed = len_data
else:
number_of_chunks, degree, seed = len_data
self.number_of_chunks = xor_mask(number_of_chunks, number_of_chunks_len_format)
else:
if self.implicit_mode:
seed = len_data
else:
degree, seed = len_data
seed: int = xor_mask(seed, seed_len_format)
if degree is None:
self.dist.set_seed(seed)
degree: int = self.dist.getNumber()
else:
degree: int = xor_mask(degree, degree_len_format)
used_packets = self.choose_packet_numbers(degree, seed)
data = packet[struct_len:crc_len]
self.correct += 1
return Packet(data, used_packets, self.number_of_chunks, read_only=True, error_correction=self.error_correction,
save_number_of_chunks_in_packet=self.static_number_of_chunks is None)
def prepare_and_pack(self) -> bytes:
# Format = Highest possible Packetnumber for this file, quality settings, epsilon, and a (hopefully) unique checkBlock-Number
struct_format = "<" + (
self.number_of_chunks_len_format
if self.save_number_of_chunks_in_packet else ""
) + self.quality_len_format + self.epsilon_len_format + self.check_block_number_len_format
if self.save_number_of_chunks_in_packet:
return struct.pack(
struct_format,
xor_mask(self.total_number_of_chunks,
self.number_of_chunks_len_format),
xor_mask(self.quality, self.quality_len_format), self.epsilon,
self.check_block_number)
else:
return struct.pack(struct_format,
xor_mask(self.quality, self.quality_len_format),
self.epsilon, self.check_block_number)
def blackbox(file, number_of_chunks, seed, overhead, leng=100):
start = time.time()
name = "random_bytes"
no_packets = int(math.ceil(number_of_chunks * (1.0 + overhead)))
packets = [create_packet(leng) for _ in range(no_packets)]
result = True
correct = 0
for elem in packets:
elem = struct.pack("<I" + str(len(elem)) + "sI", xor_mask(leng), elem,
xor_mask(leng)) # simulates a crc
if should_drop_packet(elem, True):
result = False
else:
correct += 1
end = time.time() - start
return name, result, number_of_chunks, number_of_chunks - correct, round(
end, 4)
def prepare_and_pack(self):
# Format = Highest possible Packetnumber for this file, number of used Packets for this
# File and the Indizies of the used Packets
struct_str = "<" + (self.number_of_chunks_len_format if self.save_number_of_chunks_in_packet else "") + (
self.used_packets_len_format if not self.implicit_mode else "") + self.id_len_format
if self.save_number_of_chunks_in_packet:
if self.implicit_mode:
return struct.pack(struct_str, xor_mask(self.total_number_of_chunks, self.number_of_chunks_len_format),
xor_mask(self.id, self.id_len_format))
else:
return struct.pack(struct_str, xor_mask(self.total_number_of_chunks, self.number_of_chunks_len_format),
xor_mask(len(self.used_packets), self.used_packets_len_format),
xor_mask(self.id, self.id_len_format))
else:
if self.implicit_mode:
return struct.pack(struct_str, xor_mask(self.id, self.id_len_format))
else:
return struct.pack(struct_str, xor_mask(len(self.used_packets), self.used_packets_len_format),
xor_mask(self.id, self.id_len_format))
def getNextValidPacket(self, from_multiple_files: bool = False, packet_len_format: str = "I",
crc_len_format: str = "L", number_of_chunks_len_format: str = "I",
degree_len_format: str = "I", seed_len_format: str = "I",
last_chunk_len_format: str = "I") -> typing.Optional[Packet]:
if not from_multiple_files:
packet_len: typing.Union[bytes, int] = self.f.read(struct.calcsize("<" + packet_len_format))
packet_len = struct.unpack("<" + packet_len_format, packet_len)[0]
packet: bytes = self.f.read(int(packet_len))
else:
packet = self.f.read()
packet_len = len(packet)
if not packet or not packet_len: # EOF
self.EOF: bool = True
self.f.close()
return None
crc_len: typing.Optional[int] = -struct.calcsize("<" + crc_len_format)
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
payload = packet[:crc_len]
# instead of typing.Any we would have _SupportsIndex:
crc: typing.Union[int, typing.Any] = struct.unpack("<" + crc_len_format, packet[crc_len:])[0]
calced_crc = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc)))
self.corrupt += 1
return self.getNextValidPacket(from_multiple_files, packet_len_format=packet_len_format,
crc_len_format=crc_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
degree_len_format=degree_len_format, seed_len_format=seed_len_format,
last_chunk_len_format=last_chunk_len_format)
else:
crc_len = None
try:
packet = self.error_correction(packet)
except:
self.corrupt += 1
return self.getNextValidPacket(from_multiple_files)
struct_str = "<" + number_of_chunks_len_format + degree_len_format + seed_len_format
struct_len = struct.calcsize(struct_str)
len_data = struct.unpack(struct_str, packet[0:struct_len])
degree = None
if self.static_number_of_chunks is None:
if self.implicit_mode:
number_of_chunks, seed = len_data
else:
number_of_chunks, degree, seed = len_data
self.number_of_chunks = xor_mask(number_of_chunks, number_of_chunks_len_format)
else:
if self.implicit_mode:
seed, = len_data
else:
degree, seed = len_data
seed = xor_mask(seed, seed_len_format)
if degree is None:
self.dist.set_seed(seed)
degree = self.dist.getNumber()
else:
degree = xor_mask(degree, degree_len_format)
used_packets = self.choose_packet_numbers(degree, seed=seed)
data = packet[struct_len:crc_len]
self.correct += 1
res = DecodePacket(data, used_packets, error_correction=self.error_correction,
number_of_chunks=self.number_of_chunks)
if used_packets.issubset({0}) and self.headerChunk is None and self.use_headerchunk:
self.headerChunk = HeaderChunk(res)
return res
def getNextValidPacket(
self,
from_multiple_files: bool = False,
packet_len_format: str = "I",
crc_len_format: str = "L",
number_of_chunks_len_format: str = "I",
quality_len_format: str = "I",
epsilon_len_format: str = "f",
check_block_number_len_format: str = "I"
) -> typing.Optional[OnlinePacket]:
if not from_multiple_files:
packet_len = self.f.read(struct.calcsize("<" + packet_len_format))
packet_len = struct.unpack("<" + packet_len_format, packet_len)[0]
packet: bytes = self.f.read(int(packet_len))
else:
packet = self.f.read()
packet_len = len(packet)
if not packet or not packet_len: # EOF
self.EOF = True
self.f.close()
return None
crc_len: typing.Optional[int] = struct.calcsize("<" + crc_len_format)
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
payload = packet[:crc_len]
crc = struct.unpack("<L", packet[crc_len:])[0]
calced_crc = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " +
str(hex(calced_crc)))
self.corrupt += 1
return self.getNextValidPacket(from_multiple_files)
else:
crc_len = None
try:
packet = self.error_correction(packet)
except:
self.corrupt += 1
return self.getNextValidPacket(from_multiple_files)
struct_str: str = "<" + number_of_chunks_len_format + quality_len_format + epsilon_len_format + check_block_number_len_format
struct_len: int = struct.calcsize(struct_str)
data = packet[struct_len:crc_len]
len_data: typing.Union[typing.Tuple[int, float, int],
typing.Tuple[int, int, float,
int]] = struct.unpack(
struct_str,
packet[0:struct_len])
if self.static_number_of_chunks is None:
number_of_chunks, quality, self.epsilon, check_block_number = len_data
self.number_of_chunks = xor_mask(number_of_chunks,
number_of_chunks_len_format)
else:
quality, self.epsilon, check_block_number = len_data
self.quality = xor_mask(quality, quality_len_format)
if self.dist is None:
self.dist = OnlineDistribution(self.epsilon)
if self.correct == 0:
# Create MockUp AuxBlocks with the given Pseudo-Random Number -> we will know which Packets are Encoded in which AuxBlock
self.createAuxBlocks()
self.correct += 1
res = OnlinePacket(
data,
self.number_of_chunks,
self.quality,
self.epsilon,
check_block_number,
read_only=True,
crc_len_format=crc_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
quality_len_format=quality_len_format,
epsilon_len_format=epsilon_len_format,
check_block_number_len_format=check_block_number_len_format,
save_number_of_chunks_in_packet=self.static_number_of_chunks is
None)
return res
def parse_raw_packet(
self,
packet,
crc_len_format: str = "L",
number_of_chunks_len_format: str = "L",
packet_len_format: str = "I",
id_len_format: str = "L") -> typing.Union[RU10Packet, str]:
"""
Creates a RU10 packet from a raw given packet. Also checks if the packet is corrupted. If any method was used to
create packets from specific chunks, set self.use_method = True. This will treat the last byte of the raw packet
data as the byte that contains the information about the used method ("even", "odd", "window_30 + window" or
"window_40 + window". See RU10Encoder.create_new_packet_from_chunks for further information.
:param packet: A raw packet
:param packet_len_format: Format of the packet length
:param crc_len_format: Format of the crc length
:param number_of_chunks_len_format: Format of the number of chunks length
:param id_len_format: Format of the ID length
:return: RU10Packet or an error message
"""
struct_str = "<" + number_of_chunks_len_format + id_len_format
struct_len = struct.calcsize(struct_str)
""""
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
crc_len = -struct.calcsize("<" + crc_len_format)
payload = packet[:crc_len]
crc = struct.unpack("<" + crc_len_format, packet[crc_len:])[0]
calced_crc = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc)))
self.corrupt += 1
return "CORRUPT"
else:
"""
try:
packet = self.error_correction(packet)
except:
self.corrupt += 1
return "CORRUPT"
data = packet[struct_len:]
if self.use_method:
method_data = bin(data[-1])[2:]
while len(method_data) < 8:
method_data = '0' + method_data
data = data[:-1]
if method_data.startswith('00'):
chunk_lst = [
ch for ch in range(0, self.number_of_chunks + 1)
if ch % 2 == 0
]
elif method_data.startswith('01'):
chunk_lst = [
ch for ch in range(0, self.number_of_chunks + 1)
if ch % 2 != 0
]
elif method_data.startswith('10'):
window = int(method_data[2:], 2)
window_size = 30
start = window * (window_size - 10)
chunk_lst = [
ch for ch in range(start, start + window_size)
if ch <= self.number_of_chunks
]
elif method_data.startswith('11'):
window = int(method_data[2:], 2)
window_size = 40
start = window * (window_size - 10)
chunk_lst = [
ch for ch in range(start, start + window_size)
if ch <= self.number_of_chunks
]
else:
raise RuntimeError(f"Invalid method_data: %s" % method_data)
len_data = struct.unpack(struct_str, packet[0:struct_len])
if self.static_number_of_chunks is None:
self.number_of_chunks = xor_mask(len_data[0],
number_of_chunks_len_format)
unxored_id = xor_mask(len_data[1], id_len_format)
else:
unxored_id = xor_mask(len_data[0], id_len_format)
if self.dist is None:
self.dist = RaptorDistribution(self.number_of_chunks)
_, self.s, self.h = intermediate_symbols(self.number_of_chunks,
self.dist)
if self.correct == 0:
self.createAuxBlocks()
self.correct += 1
if self.use_method:
numbers = choose_packet_numbers(len(chunk_lst),
unxored_id,
self.dist,
systematic=False,
max_l=len(chunk_lst))
used_packets = set([chunk_lst[i] for i in numbers])
else:
used_packets = set(
choose_packet_numbers(self.number_of_chunks,
unxored_id,
self.dist,
systematic=False))
res = RU10Packet(
data,
used_packets,
self.number_of_chunks,
unxored_id,
read_only=True,
packet_len_format=packet_len_format,
crc_len_format=crc_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
id_len_format=id_len_format,
save_number_of_chunks_in_packet=self.static_number_of_chunks is
None)
return res
