def main(file: str,
error_correction: typing.Callable[[typing.Any], typing.Any],
asdna: bool = True,
epsilon: float = 0.06,
insert_header: bool = False):
dist = OnlineDistribution(epsilon)
number_of_chunks = dist.get_size()
quality = 7
if asdna:
rules = FastDNARules()
else:
rules = None
encoder = OnlineEncoder(file,
number_of_chunks,
dist,
epsilon,
quality,
error_correction=error_correction,
quality_len_format="B",
insert_header=insert_header,
check_block_number_len_format="H",
number_of_chunks_len_format="H",
rules=rules,
save_number_of_chunks_in_packet=False)
encoder.set_overhead_limit(1.70)
encoder.encode_file(split_to_multiple_files=True, save_as_dna=asdna)
encoder.save_packets(True, save_as_dna=asdna)
def blackboxOnlineTest(file, number_of_chunks=800, seed=2):
start = time.time()
epsilon = 0.05
quality = 7
dist = OnlineDistribution(epsilon, seed)
number_of_chunks = dist.get_size()
print("Starting Blackbox Test with " + str(number_of_chunks) + " Chunks")
pseudo = OnlineBPDecoder.pseudo_decoder(number_of_chunks)
encoder = OnlineEncoder(file,
number_of_chunks,
dist,
epsilon,
quality,
pseudo_decoder=pseudo) # , chunk_size=chunk_size)
decoder = OnlineDecoder.pseudo_decoder(number_of_chunks)
decoder.set_read_all_before_decode(True)
result, numberOfEncodedPackets, droped_count = blackbox(encoder, decoder)
end = time.time() - start
print("Blackbox-Decode " + ("successful" if result else "NOT successful") +
" after " + str(round(end, 4)) + " sec.")
return [
"Online_eps=" + str(epsilon) + "_quality=" + str(quality),
result,
numberOfEncodedPackets,
droped_count,
round(end, 4),
number_of_chunks,
]
def __init__(self,
data: bytes,
total_number_of_chunks: int,
quality: int,
epsilon: float,
check_block_number: int,
used_packets: typing.Optional[typing.Union[
typing.List[int], typing.Set[int]]] = None,
dist: typing.Optional[OnlineDistribution] = None,
read_only: bool = False,
error_correction: typing.Callable[[typing.Any],
typing.Any] = nocode,
crc_len_format: str = "L",
number_of_chunks_len_format: str = "I",
quality_len_format: str = "I",
epsilon_len_format: str = "f",
check_block_number_len_format: str = "I",
save_number_of_chunks_in_packet: bool = True,
prepend="",
append=""):
self.data: bytes = data
self.total_number_of_chunks: int = total_number_of_chunks
self.quality: int = quality
self.epsilon: float = epsilon
self.error_correction: typing.Callable[[typing.Any],
typing.Any] = error_correction
self.check_block_number: int = check_block_number
self.id: int = self.check_block_number
self.dna_data: typing.Optional[str] = None
self.used_packets: typing.Optional[typing.Set[int]] = None
self.number_of_chunks_len_format: str = number_of_chunks_len_format
self.quality_len_format: str = quality_len_format
self.epsilon_len_format: str = epsilon_len_format
self.check_block_number_len_format: str = check_block_number_len_format
self.id_len_format: str = self.check_block_number_len_format
self.crc_len_format: str = crc_len_format
self.save_number_of_chunks_in_packet: bool = save_number_of_chunks_in_packet
self.internal_hash: typing.Optional[int] = None
self.error_prob: typing.Optional[int] = None
self.packet_len_format: typing.Optional[str] = None
self.prepend = prepend # not tested for Online-Code
self.append = append # not tested for Online-Code
if not read_only:
self.packedInfo: bytes = self.prepare_and_pack()
self.packed: bytes = self.calculate_packed_data()
self.aux_number: int = -1
if dist is not None:
self.dist: OnlineDistribution = dist
else:
self.dist = OnlineDistribution(self.epsilon)
if used_packets is None:
self.calcUsedPackets()
else:
self.set_used_packets(used_packets)
self.update_degree()
def input_new_packet(self, packet: OnlinePacket) -> bool:
if self.isPseudo and self.auxBlocks == dict():
self.number_of_chunks = packet.get_total_number_of_chunks()
self.quality = packet.getQuality()
self.epsilon = round(packet.getEpsilon(), 6)
self.dist: Distribution = OnlineDistribution(self.epsilon)
self.createAuxBlocks()
removed: typing.List[typing.List[bool]] = self.removeAndXorAuxPackets(
packet)
if self.do_count:
for i in range(len(removed)):
if i in self.counter.keys():
if removed[i]:
self.counter[i] += 1
else:
self.counter[i] = 1
if self.GEPP is None:
self.GEPP: GEPP = GEPP(
numpy.array([removed], dtype=bool),
numpy.array([[packet.get_data()]], dtype=bytes),
)
else:
self.GEPP.addRow(
self.removeAndXorAuxPackets(packet),
numpy.frombuffer(packet.get_data(), dtype="uint8"),
)
if self.isPseudo and not self.read_all_before_decode and (
self.GEPP.isPotentionallySolvable() and self.GEPP.n % 25 == 0):
if self.debug:
print("current size: " + str(self.GEPP.n))
return self.GEPP.solve(partial=False)
return False
def OnlineVergleich():
for a in ["log", ""]:
dist = OnlineDistribution(eps=0.01, seed=0)
print("eps = 0.01")
print(dist.pre_comp_dist[:30])
if a == "log":
plt.semilogy([0.0] + dist.pre_comp_dist[:30],
label="$\epsilon$ = 0.01")
else:
plt.plot([0.0] + dist.pre_comp_dist[:30],
label="$\epsilon$ = 0.01")
dist = OnlineDistribution(eps=0.03, seed=0)
print("eps = 0.03")
print(dist.pre_comp_dist[:30])
if a == "log":
plt.semilogy([0.0] + dist.pre_comp_dist[:30],
label="$\epsilon$ = 0.03")
else:
plt.plot([0.0] + dist.pre_comp_dist[:30],
label="$\epsilon$ = 0.03")
dist = OnlineDistribution(eps=0.06, seed=0)
print("eps = 0.06")
print(dist.pre_comp_dist[:30])
if a == "log":
plt.semilogy([0.0] + dist.pre_comp_dist[:30],
label="$\epsilon$ = 0.06")
else:
plt.plot([0.0] + dist.pre_comp_dist[:30],
label="$\epsilon$ = 0.06")
plt.ylabel("Probability")
plt.xlabel("Degree")
plt.xticks(np.arange(0, 31, step=5))
manager = plt.get_current_fig_manager()
manager.resize(*manager.window.maxsize())
plt.grid(True)
plt.tight_layout()
plt.legend()
plt.show(block=False)
plt.savefig("../plotDists/Vergleich_Online_30" + "_" + a + ".pdf",
bbox_inches="tight")
plt.savefig("../plotDists/Vergleich_Online_30" + "_" + a + ".svg")
plt.close()
def parse_raw_packet(self, packet: bytes, crc_len_format="L", number_of_chunks_len_format="I",
quality_len_format="I", epsilon_len_format="f", check_block_number_len_format="I") -> \
typing.Optional[typing.Union[str, OnlinePacket]]:
crc_len = -struct.calcsize("<" + crc_len_format)
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
payload = packet[:crc_len]
crc: int = struct.unpack("<" + crc_len_format, packet[crc_len:])[0]
calced_crc: int = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " +
str(hex(calced_crc)))
self.corrupt += 1
return "CORRUPT"
else:
crc_len = None
try:
packet = self.error_correction(packet)
except:
return "CORRUPT" # if RS or other error correction cannot reconstruct this packet
struct_str: str = "<" + number_of_chunks_len_format + quality_len_format + epsilon_len_format + check_block_number_len_format
struct_len: int = struct.calcsize(struct_str)
data: bytes = packet[struct_len:crc_len]
len_data: typing.Union[typing.Tuple[int, int, float, int],
typing.Tuple[int, float, int]] = struct.unpack(
struct_str, packet[0:struct_len])
if self.static_number_of_chunks is None:
number_of_chunks, quality, self.epsilon, check_block_number = len_data
self.number_of_chunks = xor_mask(number_of_chunks,
number_of_chunks_len_format)
else:
quality, self.epsilon, check_block_number = len_data
self.epsilon = round(self.epsilon, 6)
self.quality = xor_mask(quality, quality_len_format)
if self.dist is None:
self.dist = OnlineDistribution(self.epsilon)
if self.correct == 0:
self.createAuxBlocks()
# Create MockUp AuxBlocks with the given Pseudo-Random Number -> we will know which Packets are Encoded in which AuxBlock
self.correct += 1
res = OnlinePacket(
data,
self.number_of_chunks,
self.quality,
self.epsilon,
check_block_number,
dist=self.dist,
read_only=True,
error_correction=self.error_correction,
crc_len_format=crc_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
quality_len_format=quality_len_format,
epsilon_len_format=epsilon_len_format,
check_block_number_len_format=check_block_number_len_format,
save_number_of_chunks_in_packet=self.static_number_of_chunks is
None)
return res
def get_err_dist(_method, _number_of_chunks, _repair_symbols):
if _method == 'RU10':
dist = RaptorDistribution(_number_of_chunks)
elif _method == 'LT':
dist = ErlichZielinskiRobustSolitonDistribution(_number_of_chunks,
seed=2)
elif _method == 'Online':
dist = OnlineDistribution(ONLINE_EPS)
else:
raise NotImplementedError("Choose: RU10, LT or Online")
return dist, lambda x: reed_solomon_encode(x, _repair_symbols)
def blackboxOnlineTest(file,
number_of_chunks=800,
seed=2,
overhead=0.20,
scale_first=1.0,
scale_second=1.0):
start = time.time()
epsilon = (
0.024343
) # pruefOrdnung: 0.007084 # according to filesize this should make a length of 200...
quality = 5
dist = OnlineDistribution(epsilon, seed)
number_of_chunks = dist.get_size()
algo_type.clear()
algo_type.append("Online_" + str(number_of_chunks) + "_" +
str(dist.get_config_string()))
print(bcolors.OK + "Starting Blackbox Test with " + str(number_of_chunks) +
" Chunks" + bcolors.ENDC)
encoder = OnlineEncoder(file, number_of_chunks, dist, epsilon, quality)
encoder.set_overhead_limit(overhead)
decoder = OnlineDecoder.pseudo_decoder(number_of_chunks)
decoder.set_read_all_before_decode(True)
result, dec_input, invalid_drop = blackbox(encoder,
decoder,
scale_first=scale_first,
scale_second=scale_second)
end = time.time() - start
print(bcolors.BLUE + "Blackbox-Decode " +
(bcolors.OK + "successful" if result else bcolors.ERR +
"NOT successful") + bcolors.END + bcolors.BLUE + " after " +
str(round(end, 4)) + " sec." + bcolors.ENDC)
return [
"Online_eps=" + str(epsilon) + "_quality=" + str(quality),
result,
number_of_chunks,
dec_input,
invalid_drop,
round(end, 4),
]
def onlineDist():
for a in ["log", ""]:
for eps in np.arange(0.01, 0.1, 0.01):
dist = OnlineDistribution(eps=eps, seed=0)
print(dist.pre_comp_dist)
if a == "log":
plt.semilogy([0.0] + dist.pre_comp_dist)
else:
plt.plot([0.0] + dist.pre_comp_dist)
plt.ylabel("Probability")
plt.xlabel("Degree")
manager = plt.get_current_fig_manager()
manager.resize(*manager.window.maxsize())
plt.grid(True)
plt.tight_layout()
plt.show(block=False)
plt.savefig(
"../plotDists/Online_eps" + str(eps) + "_" + a + ".pdf",
bbox_inches="tight",
)
plt.savefig("../plotDists/Online_eps" + str(eps) + "_" + a +
".svg")
plt.close()
def main(file):
print("###### LT Codec number_of_chunks = 700 ######")
print("##### Ideal Soliton Distribution #####")
print("### LT without prioritized Packets ###")
start = time.time()
number_of_chunks = 700
dist = IdealSolitonDistribution(number_of_chunks, seed=2)
encoder = LTEncoder(file, number_of_chunks, dist)
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### LT without prioritized Packets with pseudo_decoder ###")
start = time.time()
number_of_chunks = 700
dist = IdealSolitonDistribution(number_of_chunks, seed=2)
pseudo = LTBPDecoder.pseudo_decoder(number_of_chunks)
encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo)
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### LT with prioritized Packets ###")
start = time.time()
number_of_chunks = 700
dist = IdealSolitonDistribution(number_of_chunks, seed=2)
encoder = LTEncoder(file,
number_of_chunks,
dist,
prioritized_packets=[1, 3, 5])
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### LT with prioritized Packets AND pseudo_decoder ###")
start = time.time()
number_of_chunks = 700
dist = IdealSolitonDistribution(number_of_chunks, seed=2)
pseudo = LTBPDecoder.pseudo_decoder(number_of_chunks)
encoder = LTEncoder(file,
number_of_chunks,
dist,
pseudo_decoder=pseudo,
prioritized_packets=[1, 3, 5])
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### LT with prioritized Packets AND Gauss-pseudo_decoder ###")
start = time.time()
number_of_chunks = 700
dist = IdealSolitonDistribution(number_of_chunks, seed=2)
pseudo = LTDecoder.pseudo_decoder(number_of_chunks)
encoder = LTEncoder(file,
number_of_chunks,
dist,
pseudo_decoder=pseudo,
prioritized_packets=[1, 3, 5])
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("##### Robust Soliton Distribution #####")
print("### LT without prioritized Packets ###")
start = time.time()
number_of_chunks = 700
dist = RobustSolitonDistribution(number_of_chunks, seed=2)
encoder = LTEncoder(file, number_of_chunks, dist)
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### LT without prioritized Packets with Gauss-pseudo_decoder ###")
start = time.time()
number_of_chunks = 700
dist = RobustSolitonDistribution(number_of_chunks, seed=2)
pseudo = LTDecoder.pseudo_decoder(number_of_chunks)
encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo)
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### LT with prioritized Packets ###")
start = time.time()
number_of_chunks = 700
dist = RobustSolitonDistribution(number_of_chunks, seed=2)
encoder = LTEncoder(file,
number_of_chunks,
dist,
prioritized_packets=[1, 3, 5])
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### LT with prioritized Packets AND Gauss-pseudo_decoder ###")
start = time.time()
number_of_chunks = 700
dist = RobustSolitonDistribution(number_of_chunks, seed=2)
pseudo = LTDecoder.pseudo_decoder(number_of_chunks)
encoder = LTEncoder(file,
number_of_chunks,
dist,
pseudo_decoder=pseudo,
prioritized_packets=[1, 3, 5])
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print(
"###### Online Codec - number_of_chunks = 2500, eps=0.01, quality=3 ######"
)
print("### Online without pseudo_decoder ###")
start = time.time()
number_of_chunks = 2500
epsilon = 0.01
quality = 3
dist = OnlineDistribution(epsilon)
# infer number_of_chunks form Distribution:
number_of_chunks = dist.get_size()
pseudo = OnlineBPDecoder.pseudo_decoder()
encoder = OnlineEncoder(file, number_of_chunks, dist, epsilon, quality)
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
print("### Online with pseudo_decoder ###")
start = time.time()
number_of_chunks = 2500
epsilon = 0.01
quality = 3
dist = OnlineDistribution(epsilon)
# infer number_of_chunks form Distribution:
number_of_chunks = dist.get_size()
pseudo = OnlineBPDecoder.pseudo_decoder()
encoder = OnlineEncoder(file,
number_of_chunks,
dist,
epsilon,
quality,
pseudo_decoder=pseudo)
encoder.encode_to_packets()
end = time.time() - start
print("Finished encoding after " + str(round(end, 4)) + " sec. " +
str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
def getNextValidPacket(
self,
from_multiple_files: bool = False,
packet_len_format: str = "I",
crc_len_format: str = "L",
number_of_chunks_len_format: str = "I",
quality_len_format: str = "I",
epsilon_len_format: str = "f",
check_block_number_len_format: str = "I"
) -> typing.Optional[OnlinePacket]:
if not from_multiple_files:
packet_len = self.f.read(struct.calcsize("<" + packet_len_format))
packet_len = struct.unpack("<" + packet_len_format, packet_len)[0]
packet: bytes = self.f.read(int(packet_len))
else:
packet = self.f.read()
packet_len = len(packet)
if not packet or not packet_len: # EOF
self.EOF = True
self.f.close()
return None
crc_len: typing.Optional[int] = struct.calcsize("<" + crc_len_format)
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
payload = packet[:crc_len]
crc = struct.unpack("<L", packet[crc_len:])[0]
calced_crc = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " +
str(hex(calced_crc)))
self.corrupt += 1
return self.getNextValidPacket(from_multiple_files)
else:
crc_len = None
try:
packet = self.error_correction(packet)
except:
self.corrupt += 1
return self.getNextValidPacket(from_multiple_files)
struct_str: str = "<" + number_of_chunks_len_format + quality_len_format + epsilon_len_format + check_block_number_len_format
struct_len: int = struct.calcsize(struct_str)
data = packet[struct_len:crc_len]
len_data: typing.Union[typing.Tuple[int, float, int],
typing.Tuple[int, int, float,
int]] = struct.unpack(
struct_str,
packet[0:struct_len])
if self.static_number_of_chunks is None:
number_of_chunks, quality, self.epsilon, check_block_number = len_data
self.number_of_chunks = xor_mask(number_of_chunks,
number_of_chunks_len_format)
else:
quality, self.epsilon, check_block_number = len_data
self.quality = xor_mask(quality, quality_len_format)
if self.dist is None:
self.dist = OnlineDistribution(self.epsilon)
if self.correct == 0:
# Create MockUp AuxBlocks with the given Pseudo-Random Number -> we will know which Packets are Encoded in which AuxBlock
self.createAuxBlocks()
self.correct += 1
res = OnlinePacket(
data,
self.number_of_chunks,
self.quality,
self.epsilon,
check_block_number,
read_only=True,
crc_len_format=crc_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
quality_len_format=quality_len_format,
epsilon_len_format=epsilon_len_format,
check_block_number_len_format=check_block_number_len_format,
save_number_of_chunks_in_packet=self.static_number_of_chunks is
None)
return res
class OnlinePacket(Packet):
def __init__(self,
data: bytes,
total_number_of_chunks: int,
quality: int,
epsilon: float,
check_block_number: int,
used_packets: typing.Optional[typing.Union[
typing.List[int], typing.Set[int]]] = None,
dist: typing.Optional[OnlineDistribution] = None,
read_only: bool = False,
error_correction: typing.Callable[[typing.Any],
typing.Any] = nocode,
crc_len_format: str = "L",
number_of_chunks_len_format: str = "I",
quality_len_format: str = "I",
epsilon_len_format: str = "f",
check_block_number_len_format: str = "I",
save_number_of_chunks_in_packet: bool = True,
prepend="",
append=""):
self.data: bytes = data
self.total_number_of_chunks: int = total_number_of_chunks
self.quality: int = quality
self.epsilon: float = epsilon
self.error_correction: typing.Callable[[typing.Any],
typing.Any] = error_correction
self.check_block_number: int = check_block_number
self.id: int = self.check_block_number
self.dna_data: typing.Optional[str] = None
self.used_packets: typing.Optional[typing.Set[int]] = None
self.number_of_chunks_len_format: str = number_of_chunks_len_format
self.quality_len_format: str = quality_len_format
self.epsilon_len_format: str = epsilon_len_format
self.check_block_number_len_format: str = check_block_number_len_format
self.id_len_format: str = self.check_block_number_len_format
self.crc_len_format: str = crc_len_format
self.save_number_of_chunks_in_packet: bool = save_number_of_chunks_in_packet
self.internal_hash: typing.Optional[int] = None
self.error_prob: typing.Optional[int] = None
self.packet_len_format: typing.Optional[str] = None
self.prepend = prepend # not tested for Online-Code
self.append = append # not tested for Online-Code
if not read_only:
self.packedInfo: bytes = self.prepare_and_pack()
self.packed: bytes = self.calculate_packed_data()
self.aux_number: int = -1
if dist is not None:
self.dist: OnlineDistribution = dist
else:
self.dist = OnlineDistribution(self.epsilon)
if used_packets is None:
self.calcUsedPackets()
else:
self.set_used_packets(used_packets)
self.update_degree()
def getQuality(self) -> int:
return self.quality
def getEpsilon(self) -> float:
return self.epsilon
def prepare_and_pack(self) -> bytes:
# Format = Highest possible Packetnumber for this file, quality settings, epsilon, and a (hopefully) unique checkBlock-Number
struct_format = "<" + (
self.number_of_chunks_len_format
if self.save_number_of_chunks_in_packet else ""
) + self.quality_len_format + self.epsilon_len_format + self.check_block_number_len_format
if self.save_number_of_chunks_in_packet:
return struct.pack(
struct_format,
xor_mask(self.total_number_of_chunks,
self.number_of_chunks_len_format),
xor_mask(self.quality, self.quality_len_format), self.epsilon,
self.check_block_number)
else:
return struct.pack(struct_format,
xor_mask(self.quality, self.quality_len_format),
self.epsilon, self.check_block_number)
def calculate_packed_data(self) -> bytes:
# Länge des Packets + Infos + Data + crc
payload = struct.pack(
"<" + str(len(self.packedInfo)) + "s" + str(len(self.data)) + "s",
self.packedInfo, bytes(self.data))
return self.error_correction(payload)
def get_data(self) -> bytes:
return self.data
def set_used_packets(self, used_packets: typing.Set[int]):
self.used_packets = used_packets
self.internal_hash = hash(frozenset(i for i in self.used_packets))
self.update_degree()
def calcUsedPackets(self):
self.dist.set_seed(self.check_block_number)
degree: int = self.dist.getNumber()
self.set_used_packets(
self.choose_packet_numbers(degree, self.check_block_number))
def choose_packet_numbers(self, degree: int, seed: int) -> typing.Set[int]:
rng = np.random
rng.seed(seed)
res: typing.Set[int] = set()
for _ in range(0, degree):
tmp = rng.choice(
range(
0, self.total_number_of_chunks +
self.getNumberOfAuxBlocks())) # +1 for HeaderChunk
while tmp in res:
tmp = rng.choice(
range(
0, self.total_number_of_chunks +
self.getNumberOfAuxBlocks())) # +1 for HeaderChunk
res.add(tmp)
return res
def getNumberOfAuxBlocks(self) -> int:
return ceil(0.55 * self.quality * self.epsilon *
self.total_number_of_chunks)
def get_bool_array_used_packets(self) -> typing.List[bool]:
return [
x in self.used_packets for x in range(self.total_number_of_chunks)
]
def getBoolArrayAuxPackets(self) -> typing.List[bool]:
return [
x in self.used_packets for x in range(
self.total_number_of_chunks,
self.total_number_of_chunks + self.getNumberOfAuxBlocks(),
)
]
def __str__(self) -> str:
return ("< used_packets: " + str(self.used_packets) + " , Data: " +
str(self.data) + " , Error Correction: " +
str(self.error_correction) + " >")
def __hash__(self) -> int:
return hash(frozenset(hash(i) for i in self.used_packets))
def test_suite(as_dna, decoder_instance, use_header):
dir_path = os.getcwd()
try:
os.remove(dir_path + "/" + file)
except:
print("Not deleting, File did not exists")
shutil.copyfile(dir_path + "/" + cmp_file, dir_path + "/" + file)
chunksize = 200
epsilon = 0.07
quality = 5
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunksize)
dist = OnlineDistribution(epsilon)
pseudo_decoder = decoder_instance.pseudo_decoder(
number_of_chunks=number_of_chunks)
rules = FastDNARules() if as_dna else None
encoder = OnlineEncoder(file,
number_of_chunks,
dist,
epsilon,
quality,
number_of_chunks_len_format="H",
check_block_number_len_format="H",
quality_len_format="B",
pseudo_decoder=pseudo_decoder,
rules=rules,
insert_header=use_header)
encoder.encode_to_packets()
encoder.save_packets(split_to_multiple_files=True, save_as_dna=as_dna)
assert (pseudo_decoder.is_decoded() and pseudo_decoder.getSolvedCount()
== pseudo_decoder.number_of_chunks)
assert os.path.exists(out_dir)
decoder = decoder_instance(out_dir, use_headerchunk=use_header)
decoder.decodeFolder(number_of_chunks_len_format="H",
check_block_number_len_format="H",
quality_len_format="B")
if decoder_instance == OnlineBPDecoder:
decoder.decodeFolder(number_of_chunks_len_format="H",
check_block_number_len_format="H",
quality_len_format="B")
decoder.solve()
assert decoder.is_decoded() and decoder.getSolvedCount(
) == encoder.number_of_chunks
if not use_header:
out_file = "DEC_ONLINE_" + file
else:
out_file = file
try:
os.remove(out_file)
except:
print("Not deleting, File did not exists")
decoder.saveDecodedFile(print_to_output=False)
assert os.path.exists(out_file) and filecmp.cmp(out_file, cmp_file)
if decoder_instance == OnlineBPDecoder:
# since ApproxDecoder defines an upper bound Gauss-Decoder MUST be able to decode!
decoder = OnlineDecoder(out_dir, use_headerchunk=use_header)
decoder.decodeFolder(number_of_chunks_len_format="H",
check_block_number_len_format="H",
quality_len_format="B")
assert decoder.is_decoded() and decoder.getSolvedCount(
) == encoder.number_of_chunks
os.remove(out_file)
decoder.saveDecodedFile(print_to_output=False)
assert os.path.exists(out_file) and filecmp.cmp(file, cmp_file)
shutil.rmtree(out_dir)