def parse_raw_packet(self, packet: bytes, crc_len_format="L", number_of_chunks_len_format="I", quality_len_format="I", epsilon_len_format="f", check_block_number_len_format="I") -> \ typing.Optional[typing.Union[str, OnlinePacket]]: crc_len = -struct.calcsize("<" + crc_len_format) if self.error_correction.__code__.co_name == crc32.__code__.co_name: payload = packet[:crc_len] crc: int = struct.unpack("<" + crc_len_format, packet[crc_len:])[0] calced_crc: int = calc_crc(payload) if crc != calced_crc: # If the Packet is corrupt, try next one print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc))) self.corrupt += 1 return "CORRUPT" else: crc_len = None try: packet = self.error_correction(packet) except: return "CORRUPT" # if RS or other error correction cannot reconstruct this packet struct_str: str = "<" + number_of_chunks_len_format + quality_len_format + epsilon_len_format + check_block_number_len_format struct_len: int = struct.calcsize(struct_str) data: bytes = packet[struct_len:crc_len] len_data: typing.Union[typing.Tuple[int, int, float, int], typing.Tuple[int, float, int]] = struct.unpack( struct_str, packet[0:struct_len]) if self.static_number_of_chunks is None: number_of_chunks, quality, self.epsilon, check_block_number = len_data self.number_of_chunks = xor_mask(number_of_chunks, number_of_chunks_len_format) else: quality, self.epsilon, check_block_number = len_data self.epsilon = round(self.epsilon, 6) self.quality = xor_mask(quality, quality_len_format) if self.dist is None: self.dist = OnlineDistribution(self.epsilon) if self.correct == 0: self.createAuxBlocks() # Create MockUp AuxBlocks with the given Pseudo-Random Number -> we will know which Packets are Encoded in which AuxBlock self.correct += 1 res = OnlinePacket( data, self.number_of_chunks, self.quality, self.epsilon, check_block_number, dist=self.dist, read_only=True, error_correction=self.error_correction, crc_len_format=crc_len_format, number_of_chunks_len_format=number_of_chunks_len_format, quality_len_format=quality_len_format, epsilon_len_format=epsilon_len_format, check_block_number_len_format=check_block_number_len_format, save_number_of_chunks_in_packet=self.static_number_of_chunks is None) return res
def prepare_and_pack(self) -> bytes: # Format = Highest possible Packetnumber for this file, # number of used Packets for this File and the seed for the Indices of the used Packets if self.save_number_of_chunks_in_packet: return struct.pack( "<" + self.number_of_chunks_len_format + self.id_len_format, xor_mask(self.total_number_of_chunks, self.number_of_chunks_len_format), xor_mask(self.id, self.id_len_format)) else: return struct.pack("<" + self.id_len_format, xor_mask(self.id, self.id_len_format))
def listen(sock, queue, signals): if sock is None: return None while not signals["shutdown"]: try: data, sender = sock.recvfrom(1024) a, b = struct.unpack("<II", data[0:8]) print("Packet from: %s:%s to %s - #Chunks: %s - Id: %s" % (sender[0], sender[1], sock.getsockname()[0], xor_mask(a), xor_mask(b))) queue.put(data) except socket.error as ex: # queue.put(e) continue
def parse_raw_packet(self, packet: bytes, crc_len_format: str = "L", number_of_chunks_len_format: str = "I", degree_len_format: str = "I", seed_len_format: str = "I") -> typing.Union[str, Packet]: crc_len = -struct.calcsize("<" + crc_len_format) if self.error_correction.__code__.co_name == crc32.__code__.co_name: payload: bytes = packet[:crc_len] crc: int = struct.unpack("<" + crc_len_format, packet[crc_len:])[0] calced_crc: int = calc_crc(payload) if crc != calced_crc: # If the Packet is corrupt, try next one print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc))) self.corrupt += 1 return "CORRUPT" else: crc_len = None try: packet = self.error_correction(packet) except: self.corrupt += 1 return "CORRUPT" if self.implicit_mode: degree_len_format = "" struct_str: str = "<" + number_of_chunks_len_format + degree_len_format + seed_len_format struct_len: int = struct.calcsize(struct_str) len_data: typing.Union[int, typing.Tuple[int, int], typing.Tuple[int, int, int]] = struct.unpack(struct_str, packet[ 0:struct_len]) degree: typing.Optional[int] = None if self.static_number_of_chunks is None: if self.implicit_mode: number_of_chunks, seed = len_data else: number_of_chunks, degree, seed = len_data self.number_of_chunks = xor_mask(number_of_chunks, number_of_chunks_len_format) else: if self.implicit_mode: seed = len_data else: degree, seed = len_data seed: int = xor_mask(seed, seed_len_format) if degree is None: self.dist.set_seed(seed) degree: int = self.dist.getNumber() else: degree: int = xor_mask(degree, degree_len_format) used_packets = self.choose_packet_numbers(degree, seed) data = packet[struct_len:crc_len] self.correct += 1 return Packet(data, used_packets, self.number_of_chunks, read_only=True, error_correction=self.error_correction, save_number_of_chunks_in_packet=self.static_number_of_chunks is None)
def prepare_and_pack(self) -> bytes: # Format = Highest possible Packetnumber for this file, quality settings, epsilon, and a (hopefully) unique checkBlock-Number struct_format = "<" + ( self.number_of_chunks_len_format if self.save_number_of_chunks_in_packet else "" ) + self.quality_len_format + self.epsilon_len_format + self.check_block_number_len_format if self.save_number_of_chunks_in_packet: return struct.pack( struct_format, xor_mask(self.total_number_of_chunks, self.number_of_chunks_len_format), xor_mask(self.quality, self.quality_len_format), self.epsilon, self.check_block_number) else: return struct.pack(struct_format, xor_mask(self.quality, self.quality_len_format), self.epsilon, self.check_block_number)
def blackbox(file, number_of_chunks, seed, overhead, leng=100): start = time.time() name = "random_bytes" no_packets = int(math.ceil(number_of_chunks * (1.0 + overhead))) packets = [create_packet(leng) for _ in range(no_packets)] result = True correct = 0 for elem in packets: elem = struct.pack("<I" + str(len(elem)) + "sI", xor_mask(leng), elem, xor_mask(leng)) # simulates a crc if should_drop_packet(elem, True): result = False else: correct += 1 end = time.time() - start return name, result, number_of_chunks, number_of_chunks - correct, round( end, 4)
def prepare_and_pack(self): # Format = Highest possible Packetnumber for this file, number of used Packets for this # File and the Indizies of the used Packets struct_str = "<" + (self.number_of_chunks_len_format if self.save_number_of_chunks_in_packet else "") + ( self.used_packets_len_format if not self.implicit_mode else "") + self.id_len_format if self.save_number_of_chunks_in_packet: if self.implicit_mode: return struct.pack(struct_str, xor_mask(self.total_number_of_chunks, self.number_of_chunks_len_format), xor_mask(self.id, self.id_len_format)) else: return struct.pack(struct_str, xor_mask(self.total_number_of_chunks, self.number_of_chunks_len_format), xor_mask(len(self.used_packets), self.used_packets_len_format), xor_mask(self.id, self.id_len_format)) else: if self.implicit_mode: return struct.pack(struct_str, xor_mask(self.id, self.id_len_format)) else: return struct.pack(struct_str, xor_mask(len(self.used_packets), self.used_packets_len_format), xor_mask(self.id, self.id_len_format))
def getNextValidPacket(self, from_multiple_files: bool = False, packet_len_format: str = "I", crc_len_format: str = "L", number_of_chunks_len_format: str = "I", degree_len_format: str = "I", seed_len_format: str = "I", last_chunk_len_format: str = "I") -> typing.Optional[Packet]: if not from_multiple_files: packet_len: typing.Union[bytes, int] = self.f.read(struct.calcsize("<" + packet_len_format)) packet_len = struct.unpack("<" + packet_len_format, packet_len)[0] packet: bytes = self.f.read(int(packet_len)) else: packet = self.f.read() packet_len = len(packet) if not packet or not packet_len: # EOF self.EOF: bool = True self.f.close() return None crc_len: typing.Optional[int] = -struct.calcsize("<" + crc_len_format) if self.error_correction.__code__.co_name == crc32.__code__.co_name: payload = packet[:crc_len] # instead of typing.Any we would have _SupportsIndex: crc: typing.Union[int, typing.Any] = struct.unpack("<" + crc_len_format, packet[crc_len:])[0] calced_crc = calc_crc(payload) if crc != calced_crc: # If the Packet is corrupt, try next one print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc))) self.corrupt += 1 return self.getNextValidPacket(from_multiple_files, packet_len_format=packet_len_format, crc_len_format=crc_len_format, number_of_chunks_len_format=number_of_chunks_len_format, degree_len_format=degree_len_format, seed_len_format=seed_len_format, last_chunk_len_format=last_chunk_len_format) else: crc_len = None try: packet = self.error_correction(packet) except: self.corrupt += 1 return self.getNextValidPacket(from_multiple_files) struct_str = "<" + number_of_chunks_len_format + degree_len_format + seed_len_format struct_len = struct.calcsize(struct_str) len_data = struct.unpack(struct_str, packet[0:struct_len]) degree = None if self.static_number_of_chunks is None: if self.implicit_mode: number_of_chunks, seed = len_data else: number_of_chunks, degree, seed = len_data self.number_of_chunks = xor_mask(number_of_chunks, number_of_chunks_len_format) else: if self.implicit_mode: seed, = len_data else: degree, seed = len_data seed = xor_mask(seed, seed_len_format) if degree is None: self.dist.set_seed(seed) degree = self.dist.getNumber() else: degree = xor_mask(degree, degree_len_format) used_packets = self.choose_packet_numbers(degree, seed=seed) data = packet[struct_len:crc_len] self.correct += 1 res = DecodePacket(data, used_packets, error_correction=self.error_correction, number_of_chunks=self.number_of_chunks) if used_packets.issubset({0}) and self.headerChunk is None and self.use_headerchunk: self.headerChunk = HeaderChunk(res) return res
def getNextValidPacket( self, from_multiple_files: bool = False, packet_len_format: str = "I", crc_len_format: str = "L", number_of_chunks_len_format: str = "I", quality_len_format: str = "I", epsilon_len_format: str = "f", check_block_number_len_format: str = "I" ) -> typing.Optional[OnlinePacket]: if not from_multiple_files: packet_len = self.f.read(struct.calcsize("<" + packet_len_format)) packet_len = struct.unpack("<" + packet_len_format, packet_len)[0] packet: bytes = self.f.read(int(packet_len)) else: packet = self.f.read() packet_len = len(packet) if not packet or not packet_len: # EOF self.EOF = True self.f.close() return None crc_len: typing.Optional[int] = struct.calcsize("<" + crc_len_format) if self.error_correction.__code__.co_name == crc32.__code__.co_name: payload = packet[:crc_len] crc = struct.unpack("<L", packet[crc_len:])[0] calced_crc = calc_crc(payload) if crc != calced_crc: # If the Packet is corrupt, try next one print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc))) self.corrupt += 1 return self.getNextValidPacket(from_multiple_files) else: crc_len = None try: packet = self.error_correction(packet) except: self.corrupt += 1 return self.getNextValidPacket(from_multiple_files) struct_str: str = "<" + number_of_chunks_len_format + quality_len_format + epsilon_len_format + check_block_number_len_format struct_len: int = struct.calcsize(struct_str) data = packet[struct_len:crc_len] len_data: typing.Union[typing.Tuple[int, float, int], typing.Tuple[int, int, float, int]] = struct.unpack( struct_str, packet[0:struct_len]) if self.static_number_of_chunks is None: number_of_chunks, quality, self.epsilon, check_block_number = len_data self.number_of_chunks = xor_mask(number_of_chunks, number_of_chunks_len_format) else: quality, self.epsilon, check_block_number = len_data self.quality = xor_mask(quality, quality_len_format) if self.dist is None: self.dist = OnlineDistribution(self.epsilon) if self.correct == 0: # Create MockUp AuxBlocks with the given Pseudo-Random Number -> we will know which Packets are Encoded in which AuxBlock self.createAuxBlocks() self.correct += 1 res = OnlinePacket( data, self.number_of_chunks, self.quality, self.epsilon, check_block_number, read_only=True, crc_len_format=crc_len_format, number_of_chunks_len_format=number_of_chunks_len_format, quality_len_format=quality_len_format, epsilon_len_format=epsilon_len_format, check_block_number_len_format=check_block_number_len_format, save_number_of_chunks_in_packet=self.static_number_of_chunks is None) return res
def parse_raw_packet( self, packet, crc_len_format: str = "L", number_of_chunks_len_format: str = "L", packet_len_format: str = "I", id_len_format: str = "L") -> typing.Union[RU10Packet, str]: """ Creates a RU10 packet from a raw given packet. Also checks if the packet is corrupted. If any method was used to create packets from specific chunks, set self.use_method = True. This will treat the last byte of the raw packet data as the byte that contains the information about the used method ("even", "odd", "window_30 + window" or "window_40 + window". See RU10Encoder.create_new_packet_from_chunks for further information. :param packet: A raw packet :param packet_len_format: Format of the packet length :param crc_len_format: Format of the crc length :param number_of_chunks_len_format: Format of the number of chunks length :param id_len_format: Format of the ID length :return: RU10Packet or an error message """ struct_str = "<" + number_of_chunks_len_format + id_len_format struct_len = struct.calcsize(struct_str) """" if self.error_correction.__code__.co_name == crc32.__code__.co_name: crc_len = -struct.calcsize("<" + crc_len_format) payload = packet[:crc_len] crc = struct.unpack("<" + crc_len_format, packet[crc_len:])[0] calced_crc = calc_crc(payload) if crc != calced_crc: # If the Packet is corrupt, try next one print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc))) self.corrupt += 1 return "CORRUPT" else: """ try: packet = self.error_correction(packet) except: self.corrupt += 1 return "CORRUPT" data = packet[struct_len:] if self.use_method: method_data = bin(data[-1])[2:] while len(method_data) < 8: method_data = '0' + method_data data = data[:-1] if method_data.startswith('00'): chunk_lst = [ ch for ch in range(0, self.number_of_chunks + 1) if ch % 2 == 0 ] elif method_data.startswith('01'): chunk_lst = [ ch for ch in range(0, self.number_of_chunks + 1) if ch % 2 != 0 ] elif method_data.startswith('10'): window = int(method_data[2:], 2) window_size = 30 start = window * (window_size - 10) chunk_lst = [ ch for ch in range(start, start + window_size) if ch <= self.number_of_chunks ] elif method_data.startswith('11'): window = int(method_data[2:], 2) window_size = 40 start = window * (window_size - 10) chunk_lst = [ ch for ch in range(start, start + window_size) if ch <= self.number_of_chunks ] else: raise RuntimeError(f"Invalid method_data: %s" % method_data) len_data = struct.unpack(struct_str, packet[0:struct_len]) if self.static_number_of_chunks is None: self.number_of_chunks = xor_mask(len_data[0], number_of_chunks_len_format) unxored_id = xor_mask(len_data[1], id_len_format) else: unxored_id = xor_mask(len_data[0], id_len_format) if self.dist is None: self.dist = RaptorDistribution(self.number_of_chunks) _, self.s, self.h = intermediate_symbols(self.number_of_chunks, self.dist) if self.correct == 0: self.createAuxBlocks() self.correct += 1 if self.use_method: numbers = choose_packet_numbers(len(chunk_lst), unxored_id, self.dist, systematic=False, max_l=len(chunk_lst)) used_packets = set([chunk_lst[i] for i in numbers]) else: used_packets = set( choose_packet_numbers(self.number_of_chunks, unxored_id, self.dist, systematic=False)) res = RU10Packet( data, used_packets, self.number_of_chunks, unxored_id, read_only=True, packet_len_format=packet_len_format, crc_len_format=crc_len_format, number_of_chunks_len_format=number_of_chunks_len_format, id_len_format=id_len_format, save_number_of_chunks_in_packet=self.static_number_of_chunks is None) return res