def blackboxRU10Test(file,
number_of_chunks=800,
droprate=0.02,
seed=2,
chunk_size=200,
overhead=0.05):
print("Starting Blackbox Test with " + str(number_of_chunks) + " Chunks")
start = time.time()
dist = RaptorDistribution(number_of_chunks)
encoder = RU10Encoder(file, number_of_chunks,
dist) # , chunk_size=chunk_size)
decoder = RU10Decoder.pseudo_decoder(number_of_chunks,
read_all_before_decode=True)
encoder.set_overhead_limit(overhead)
result, numberOfEncodedPackets, droped_count, solved_count = blackbox(
encoder, decoder, droprate=droprate)
end = time.time() - start
print("Blackbox-Decode " + ("successful" if result else "NOT successful") +
" after " + str(round(end, 4)) + " sec.")
return [
dist.get_config_string(),
result,
numberOfEncodedPackets,
droped_count,
solved_count,
round(end, 4),
number_of_chunks,
]
def blackboxRU10Test(file,
number_of_chunks=800,
seed=2,
chunk_size=200,
overhead=0.20,
scale=1.0):
print(bcolors.OK + "Starting Blackbox Test with " + str(number_of_chunks) +
" Chunks" + bcolors.ENDC)
start = time.time()
dist = RaptorDistribution(number_of_chunks)
algo_type.clear()
algo_type.append("RU10_" + str(number_of_chunks) + "_" +
str(dist.get_config_string()))
encoder = RU10Encoder(file, number_of_chunks, dist, chunk_size=chunk_size)
encoder.set_overhead_limit(overhead)
decoder = RU10Decoder.pseudo_decoder(number_of_chunks)
decoder.set_read_all_before_decode(True)
result, dec_input, invalid_drop = blackbox(encoder, decoder, scale=scale)
end = time.time() - start
print(bcolors.BLUE + "Blackbox-Decode " +
(bcolors.OK + "successful" if result else bcolors.ERR +
"NOT successful") + bcolors.END + bcolors.BLUE + " after " +
str(round(end, 4)) + " sec." + bcolors.ENDC)
return [
dist.get_config_string(),
result,
number_of_chunks,
dec_input,
invalid_drop,
round(end, 4),
]
def encode(file, dist_lst, asdna=True, chunk_size=50):
"""
:param file:
:param dist_lst:
:param asdna:
:param chunk_size:
:return:
"""
packets_needed = 0
packets = dict()
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
dist.f = dist_lst
d = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40]
dist.d = d
dna_rules = FastDNARules()
if asdna:
rules = dna_rules
else:
rules = None
x = RU10Encoder(file, number_of_chunks, dist, chunk_size=chunk_size, insert_header=False, rules=rules,
error_correction=nocode, id_len_format="H", number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=False, mode_1_bmp=False)
x.prepare()
y = RU10Decoder.pseudo_decoder(x.number_of_chunks, False)
if y.distribution is None: # self.isPseudo and
y.distribution = RaptorDistribution(x.number_of_chunks)
y.distribution.f = dist_lst
y.distribution.d = d
y.number_of_chunks = x.number_of_chunks
_, y.s, y.h = intermediate_symbols(x.number_of_chunks, y.distribution)
y.createAuxBlocks()
n = 0
for p_tmp in range(45):
packets[p_tmp] = list()
while n < number_of_chunks * 50:
pack = x.create_new_packet()
if packets_needed == 0:
y.input_new_packet(pack)
should_drop_packet(dna_rules, pack)
if pack.get_degree() not in packets:
packets[pack.get_degree()] = list()
packets[pack.get_degree()].append(pack.error_prob)
n += 1
if n >= number_of_chunks and y.is_decoded() and packets_needed == 0:
packets_needed = n
# we dont want to break, we want to generate #chunks * XXX packets!
# break
print("Packets created: " + str(sum([len(x) for x in packets.values()])))
return packets, (packets_needed - number_of_chunks) / 100.0
def input_new_packet(self, packet: RU10Packet):
"""
Removes auxpackets (LDPC and Half) and adds the remaining data to the GEPP matrix.
:param packet: A Packet to add to the GEPP matrix
:return: True: If solved. False: Else.
"""
if self.auxBlocks == dict() and self.dist is None: # self.isPseudo and
self.dist = RaptorDistribution(self.number_of_chunks)
self.number_of_chunks = packet.get_total_number_of_chunks()
_, self.s, self.h = intermediate_symbols(self.number_of_chunks,
self.dist)
self.createAuxBlocks()
# we need to do it twice sine half symbols may contain ldpc symbols (which by definition are repair codes.)
if self.debug:
print("----")
print("Id = " + str(packet.id))
print(packet.used_packets)
removed = self.removeAndXorAuxPackets(packet)
if self.debug:
print(from_true_false_list(removed))
print(packet.get_error_correction())
print("----")
packet.set_used_packets(set(from_true_false_list(removed)))
if self.count:
for i in range(len(removed)):
if i in self.counter.keys():
if removed[i]:
self.counter[i] += 1
else:
self.counter[i] = 1
self.addPacket(packet)
return self.updatePackets(packet)
def encode(p_output,
file,
as_dna=True,
error_correction=nocode,
insert_header=False,
save_number_of_chunks_in_packet=False,
overhead=6.0,
clear_output=False):
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, CHUNK_SIZE)
dist = RaptorDistribution(number_of_chunks)
dna_rules = FastDNARules()
if as_dna:
rules = dna_rules
else:
rules = None
x = RU10Encoder(
file,
number_of_chunks,
dist,
chunk_size=CHUNK_SIZE,
insert_header=insert_header,
rules=rules,
error_correction=error_correction,
id_len_format="H",
number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet)
x.set_overhead_limit(overhead)
x.encode_to_packets()
p_output.send(
[ParallelPacket.from_packet(packet) for packet in x.encodedPackets])
p_output.send("DONE")
p_output.close()
return 0
def create_packets_e_prob(start_num: int, normed_dist: ndarray, number_of_packets: int, rules=None):
dist_obj = RaptorDistribution(__NUM_CHUNKS)
dist_obj.f = normed_dist
dist_obj.d = [x for x in range(0, 41)]
encoder = RU10Encoder(file=__FILE, number_of_chunks=__NUM_CHUNKS, distribution=dist_obj, insert_header=False)
encoder.prepare()
if rules is None:
rules = FastDNARules()
packets_e_prob = []
for i in range(start_num, start_num + number_of_packets):
packet = encoder.create_new_packet(seed=i)
should_drop_packet(rules, packet)
packets_e_prob.append(packet.error_prob)
del packet
del encoder
return packets_e_prob
def get_error_sum(file,
number_of_chunks,
chunk_size,
seq_seed=None,
while_count=1000):
max_seed = np.power(2, 8 * struct.calcsize(SEED_LEN_FORMAT))
dist = RaptorDistribution(number_of_chunks)
dna_rules = FastDNARules()
error_correction = lambda x: reed_solomon_encode(x, NO_REPAIR_SYMBOLS)
encoder = RU10Encoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=INSERT_HEADER,
rules=dna_rules,
error_correction=error_correction,
id_len_format=SEED_LEN_FORMAT,
number_of_chunks_len_format=NUMBER_OF_CHUNKS_LEN_FORMAT,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
prepend="",
append="")
encoder.prepare()
i = 0
res = []
while i < while_count:
if seq_seed is not None:
if seq_seed + i >= max_seed:
break
packet = encoder.create_new_packet(seed=seq_seed + i)
else:
packet = encoder.create_new_packet()
should_drop_packet(dna_rules, packet)
res.append(packet.error_prob)
return res
def test_suite5(as_dna, chunk_size, dna_rules, error_correction, headerchunk,
decoder_instance):
chunksize = chunk_size
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file2, chunksize)
dist = RaptorDistribution(number_of_chunks)
pseudo_decoder = decoder_instance.pseudo_decoder(
number_of_chunks=number_of_chunks)
rules = dna_rules if as_dna else None
encoder = RU10Encoder(file2,
number_of_chunks,
dist,
pseudo_decoder=pseudo_decoder,
rules=rules,
error_correction=error_correction,
insert_header=headerchunk)
encoder.encode_to_packets()
encoder.encode_to_packets()
encoder.save_packets(split_to_multiple_files=True, save_as_dna=as_dna)
assert (pseudo_decoder.is_decoded() and pseudo_decoder.getSolvedCount()
== pseudo_decoder.number_of_chunks)
assert os.path.exists(out_dir2)
decoder = decoder_instance(out_dir2,
use_headerchunk=headerchunk,
error_correction=error_correction)
decoder.decode()
assert decoder.is_decoded() and decoder.getSolvedCount(
) == encoder.number_of_chunks
os.remove(file2)
decoder.saveDecodedFile(print_to_output=True, null_is_terminator=True)
assert os.path.exists('DEC_RU10_' + file2) and filecmp.cmp(
'DEC_RU10_' + file2, cmp_file2)
shutil.rmtree(out_dir2)
def test_suite2(as_dna, chunk_size, dna_rules, error_correction_pair):
chunksize = chunk_size
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunksize)
dist = RaptorDistribution(number_of_chunks)
decoder_instance = RU10Decoder
pseudo_decoder = decoder_instance.pseudo_decoder(
number_of_chunks=number_of_chunks)
rules = dna_rules if as_dna else None
encoder = RU10Encoder(
file,
number_of_chunks,
dist,
pseudo_decoder=pseudo_decoder,
rules=rules,
error_correction=error_correction_pair[0],
)
encoder.encode_to_packets()
encoder.save_packets(split_to_multiple_files=True, save_as_dna=as_dna)
assert (pseudo_decoder.is_decoded() and pseudo_decoder.getSolvedCount()
== pseudo_decoder.number_of_chunks)
assert os.path.exists(out_dir)
decoder = decoder_instance(out_dir,
error_correction=error_correction_pair[1])
decoder.decode()
assert decoder.is_decoded() and decoder.getSolvedCount(
) == encoder.number_of_chunks
os.remove(file)
decoder.saveDecodedFile(print_to_output=False)
assert os.path.exists(file) and filecmp.cmp(file, cmp_file)
shutil.rmtree(out_dir)
def main(in_file: str,
num_chunks=0,
chunk_size=0,
as_dna=True,
err_correction: typing.Callable[[typing.Any], typing.Any] = nocode,
insert_header=False,
save_number_of_chunks_in_packet=False,
mode_1_bmp=False):
if chunk_size != 0:
num_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
in_file, chunk_size)
dist = RaptorDistribution(num_chunks)
elif num_chunks != 0:
dist = RaptorDistribution.RaptorDistribution(num_chunks)
else:
print("Aborting. Please set either chunk_size or number_of_chunks!")
return
if as_dna:
rules = FastDNARules()
else:
rules = None
x = RU10Encoder(
in_file,
num_chunks,
dist,
chunk_size=chunk_size,
insert_header=insert_header,
rules=rules,
error_correction=err_correction,
id_len_format="H",
number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
mode_1_bmp=mode_1_bmp)
x.encode_to_packets()
x.save_packets(True, save_as_dna=as_dna, seed_is_filename=False)
conf = {
'error_correction': e_correction,
'repair_symbols': norepair_symbols,
'asdna': as_dna,
'number_of_splits': 0,
'find_minimum_mode': True,
'seq_seed': False
}
x.save_config_file(conf, section_name="RU10_" + in_file)
def encode(self, file, asdna=True, error_correction=nocode, insert_header=False,
save_number_of_chunks_in_packet=False, mode_1_bmp=False, chunk_size=50):
packets_needed = 0
packets = dict()
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
dist.f = self.X
dist.d = self.d
dna_rules = FastDNARules()
if asdna:
rules = dna_rules
else:
rules = None
x = RU10Encoder(file, number_of_chunks, dist, chunk_size=chunk_size, insert_header=insert_header, rules=rules,
error_correction=error_correction, id_len_format="H", number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet, mode_1_bmp=mode_1_bmp)
x.prepare()
y = RU10Decoder.pseudo_decoder(x.number_of_chunks, False)
if y.distribution is None: # self.isPseudo and
y.distribution = RaptorDistribution(x.number_of_chunks)
y.distribution.f = self.X
y.distribution.d = self.d
y.number_of_chunks = x.number_of_chunks
_, y.s, y.h = intermediate_symbols(x.number_of_chunks, y.distribution)
y.createAuxBlocks()
n = 0
for p_tmp in range(45):
packets[p_tmp] = list()
while n < number_of_chunks * 50:
pack = x.create_new_packet()
if packets_needed == 0:
y.input_new_packet(pack)
should_drop_packet(dna_rules, pack)
if pack.get_degree() not in packets:
packets[pack.get_degree()] = list()
packets[pack.get_degree()].append(pack.error_prob)
n += 1
if n >= number_of_chunks and y.is_decoded() and packets_needed == 0:
packets_needed = n
# we dont want to break, we want to generate #chunks * XXX packets!
# break
print("Packets created: " + str(sum([len(x) for x in packets.values()])))
return packets, (packets_needed - number_of_chunks) / 100.0
def get_err_dist(_method, _number_of_chunks, _repair_symbols):
if _method == 'RU10':
dist = RaptorDistribution(_number_of_chunks)
elif _method == 'LT':
dist = ErlichZielinskiRobustSolitonDistribution(_number_of_chunks,
seed=2)
elif _method == 'Online':
dist = OnlineDistribution(ONLINE_EPS)
else:
raise NotImplementedError("Choose: RU10, LT or Online")
return dist, lambda x: reed_solomon_encode(x, _repair_symbols)
def __init__(self,
data,
used_packets: typing.Collection[int],
total_number_of_chunks,
id,
dist=None,
read_only=False,
error_correction=nocode,
packet_len_format="I",
crc_len_format="L",
number_of_chunks_len_format="L",
id_len_format="L",
save_number_of_chunks_in_packet=True,
method=None,
window=None,
prepend="",
append=""):
self.id: int = id
self.bool_arrayused_packets: typing.Optional[typing.List[bool]] = None
self.total_number_of_chunks: int = total_number_of_chunks
self.data: bytes = data
self.used_packets: typing.Optional[typing.Iterable[int]] = None
self.internal_hash: typing.Optional[int] = None
self.set_used_packets(used_packets)
self.degree: int = len(used_packets)
self.dna_data: typing.Optional[str] = None
self.packet_len_format: str = packet_len_format
self.crc_len_format: str = crc_len_format
self.number_of_chunks_len_format: str = number_of_chunks_len_format
self.id_len_format: str = id_len_format
self.error_correction: typing.Callable[[typing.Any],
typing.Any] = error_correction
self.save_number_of_chunks_in_packet: bool = save_number_of_chunks_in_packet
self.error_prob: typing.Optional[float] = None
if method:
self.method: typing.Optional[str] = method
self.window: typing.Optional[int] = window
self.packedMethod: typing.Optional[bytes] = self.packMethod()
else:
self.packedMethod = None
if not read_only and (len(self.data) > 0 or self.data != ""):
self.packed_used_packets = self.prepare_and_pack()
self.packed = self.calculate_packed_data()
else:
self.packed_used_packets = None
self.packed = None
if dist is None:
self.dist = RaptorDistribution(total_number_of_chunks)
else:
self.dist = dist
_, self.s, self.h = intermediate_symbols(total_number_of_chunks,
self.dist)
self.prepend = prepend
self.append = append
def encode(file, chunk_size, dist, as_dna=True, repeats=15):
"""
Encodes the file to packets until the pseudo decoder was able to decode it 'repeats' times with the given chunk size
and the distribution list.
:param file: File to encode.
:param chunk_size: Chunksize to use.
:param dist: The distribution to calculate the average error and overhead for.
:param as_dna: If true uses the DNA Rules.
:param repeats: Number of En-/Decoding cycles.
:return:
"""
degree_dict = {}
overhead_lst = []
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(file, chunk_size, insert_header=False)
distribution = RaptorDistribution(number_of_chunks)
distribution.f = dist
distribution.d = [x for x in range(0, 41)]
if as_dna:
rules = FastDNARules()
else:
rules = None
encoder = RU10Encoder(file, number_of_chunks, distribution, insert_header=False, rules=rules,
error_correction=nocode, id_len_format="H", number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=False, mode_1_bmp=False)
encoder.prepare()
for _ in range(0, repeats):
encoder.random_state = np.random.RandomState()
# print("Master-Seed used: " + str(encoder.random_state.get_state()[1][0]))
pseudo_decoder = create_pseudo_decoder(encoder.number_of_chunks, distribution)
needed_packets = 0
while pseudo_decoder.GEPP is None or not pseudo_decoder.is_decoded():
needed_packets += 1
packet = encoder.create_new_packet()
pseudo_decoder.input_new_packet(packet)
should_drop_packet(rules, packet)
if packet.get_degree() not in degree_dict:
degree_dict[packet.get_degree()] = list()
degree_dict[packet.get_degree()].append(min(packet.error_prob, 1.0))
overhead = (needed_packets - encoder.number_of_chunks) / 100.0
overhead_lst.append(overhead)
return sum(overhead_lst) / len(overhead_lst), degree_dict
def encode(file,
asdna=True,
chunk_size=DEFAULT_CHUNK_SIZE,
error_correction=nocode,
insert_header=False,
save_number_of_chunks_in_packet=False,
mode_1_bmp=False,
prepend="",
append="",
upper_bound=0.5,
save_as_fasta=True,
save_as_zip=True,
overhead=0.40):
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
if asdna:
rules = FastDNARules()
else:
rules = None
x = RU10Encoder(
file,
number_of_chunks,
dist,
insert_header=insert_header,
pseudo_decoder=None,
chunk_size=0,
rules=rules,
error_correction=error_correction,
packet_len_format=PACKET_LEN_FORMAT,
crc_len_format=CRC_LEN_FORMAT,
number_of_chunks_len_format=NUMBER_OF_CHUNKS_LEN_FORMAT,
id_len_format=ID_LEN_FORMAT,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
mode_1_bmp=mode_1_bmp,
prepend=prepend,
append=append,
drop_upper_bound=upper_bound)
x.set_overhead_limit(overhead)
x.encode_to_packets()
if save_as_fasta and asdna:
x.save_packets_fasta(file_ending="_RU10", seed_is_filename=True)
elif save_as_zip:
x.save_packets_zip(save_as_dna=asdna,
file_ending="_RU10",
seed_is_filename=True)
else:
x.save_packets(True,
save_as_dna=asdna,
seed_is_filename=True,
clear_output=True)
return x
def raptorDist():
for a in ["log", ""]:
dist = RaptorDistribution(1)
erg = [
dist.deg(eps) for eps in
[0, 10241, 491582, 712794, 831695, 948446, 1032189, 1048576]
]
print(erg)
if a == "log":
plt.semilogy([0.0] + erg)
else:
plt.plot([0.0] + erg)
plt.ylabel("random number")
plt.xlabel("Degree")
manager = plt.get_current_fig_manager()
manager.resize(*manager.window.maxsize())
plt.grid(True)
plt.tight_layout()
plt.show(block=False)
plt.savefig("../plotDists/raptor" + "_" + a + ".pdf",
bbox_inches="tight")
plt.savefig("../plotDists/raptor" + "_" + a + ".svg")
plt.close()
def input_new_packet(self, packet: RU10Packet):
"""
Removes auxpackets (LDPC and Half) and adds the remaining data to the GEPP matrix.
:param packet: A Packet to add to the GEPP matrix
:return: True: If solved. False: Else.
"""
if self.ldpcANDhalf == dict(
) and self.distribution is None: # self.isPseudo and
self.distribution = RaptorDistribution(self.number_of_chunks)
self.number_of_chunks = packet.get_total_number_of_chunks()
_, self.s, self.h = intermediate_symbols(self.number_of_chunks,
self.distribution)
self.createAuxBlocks()
self.progress_bar = self.create_progress_bar(
self.number_of_chunks + 0.02 * self.number_of_chunks)
# we need to do it twice sine half symbols may contain ldpc symbols (which by definition are repair codes.)
if self.debug:
print("----")
print("Id = " + str(packet.id))
print(packet.used_packets)
removed = self.removeAndXorAuxPackets(packet)
if self.debug:
print(from_true_false_list(removed))
print(packet.get_error_correction())
print("----")
if self.count:
for i in range(len(removed)):
if i in self.counter.keys():
if removed[i]:
self.counter[i] += 1
else:
self.counter[i] = 1
if self.GEPP is None:
self.GEPP = GEPP(
np.array([removed], dtype=bool),
np.array([[packet.get_data()]], dtype=bytes),
)
else:
self.GEPP.addRow(
np.array(removed, dtype=bool),
np.frombuffer(packet.get_data(), dtype="uint8"),
)
if (self.isPseudo or not self.read_all_before_decode
) and self.GEPP.isPotentionallySolvable():
# and self.GEPP.n % 5 == 0: # Nur alle 5 Packete versuch starten
if self.debug:
print("current size: " + str(self.GEPP.n))
return self.GEPP.solve(partial=False)
return False
def blackboxRU10Test(file, number_of_chunks=800, seed=2, chunk_size=200):
print("Starting Blackbox Test with " + str(number_of_chunks) + " Chunks")
start = time.time()
dist = RaptorDistribution(number_of_chunks)
pseudo = RU10Decoder.pseudo_decoder()
encoder = RU10Encoder(file,
number_of_chunks,
dist,
pseudo_decoder=pseudo,
chunk_size=chunk_size)
decoder = RU10Decoder.pseudo_decoder(number_of_chunks)
result, numberOfEncodedPackets, droped_count = blackbox(encoder, decoder)
end = time.time() - start
print("Blackbox-Decode " + ("successful" if result else "NOT successful") +
" after " + str(round(end, 4)) + " sec.")
return [
dist.get_config_string(),
result,
numberOfEncodedPackets,
droped_count,
round(end, 4),
number_of_chunks,
]
def test_suite4(as_dna, chunk_size, dna_rules, error_correction):
chunksize = chunk_size
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunksize)
dist = RaptorDistribution(number_of_chunks)
decoder_instance = RU10Decoder
pseudo_decoder = decoder_instance.pseudo_decoder(
number_of_chunks=number_of_chunks)
rules = dna_rules if as_dna else None
encoder = RU10Encoder(file,
number_of_chunks,
dist,
pseudo_decoder=pseudo_decoder,
rules=rules,
error_correction=error_correction[0])
encoder.encode_to_packets()
encoder.save_packets(split_to_multiple_files=True, save_as_dna=as_dna)
assert (pseudo_decoder.is_decoded() and pseudo_decoder.getSolvedCount()
== pseudo_decoder.number_of_chunks)
assert os.path.exists(out_dir)
# do not delete all packets (and break the last one).
# that way the GEPP inside the decoder will get initialized and we might not end in a race-condition for
# decoder.decode() sometimes raising an Exception..
for i in range(2, number_of_chunks):
tmp_path = "RU10_" + file + "/" + str(i) + ".RU10_DNA"
os.remove(tmp_path)
with open("RU10_" + file + "/0.RU10_DNA", 'rb+') as tmp_file:
# TODO we should flip bits in the middle rather than deleting 4 bytes at the end (we store crc32 / reedsolomon at the end)
tmp_file.seek(-4, os.SEEK_END)
tmp_file.truncate()
decoder = decoder_instance(out_dir, error_correction=error_correction[1])
decoder.decode()
assert decoder.corrupt == 1
assert not decoder.is_decoded()
os.remove(file)
with pytest.raises(AssertionError):
decoder.saveDecodedFile(print_to_output=False, partial_decoding=False)
assert not (os.path.exists(file) and filecmp.cmp(file, cmp_file))
shutil.rmtree(out_dir)
def test_suite(as_dna, decoder_instance):
dir_path = os.getcwd()
try:
os.remove(dir_path + "/" + file)
except:
print("Not deleting, File did not exists")
shutil.copyfile(dir_path + "/" + cmp_file, dir_path + "/" + file)
print(as_dna)
chunksize = 200
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunksize)
dist = RaptorDistribution(number_of_chunks)
pseudo_decoder = decoder_instance.pseudo_decoder(
number_of_chunks=number_of_chunks)
rules = FastDNARules() if as_dna else None
encoder = RU10Encoder(file,
number_of_chunks,
dist,
pseudo_decoder=pseudo_decoder,
rules=rules,
id_len_format="H",
number_of_chunks_len_format="H",
insert_header=True)
encoder.encode_to_packets()
encoder.save_packets(split_to_multiple_files=True, save_as_dna=as_dna)
assert (pseudo_decoder.is_decoded() and pseudo_decoder.getSolvedCount()
== pseudo_decoder.number_of_chunks)
assert os.path.exists(out_dir)
decoder = decoder_instance(out_dir)
decoder.decodeFolder(id_len_format="H", number_of_chunks_len_format="H")
if isinstance(decoder, RU10BPDecoder):
for pack in encoder.encodedPackets:
decoder.input_new_packet(pack)
assert decoder.is_decoded() and decoder.getSolvedCount(
) == encoder.number_of_chunks
os.remove(file)
decoder.saveDecodedFile(print_to_output=False)
assert os.path.exists(file) and filecmp.cmp(file, cmp_file)
shutil.rmtree(out_dir)
def run(seq_seed=None,
file='logo.jpg',
asdna=True,
insert_header=False,
error_correction=reed_solomon_encode,
save_number_of_chunks_in_packet=False,
l_size=1000,
while_count=1000,
chunk_size=0,
number_of_chunks=300,
prepend="",
append="",
seed_len_format=ID_LEN_FORMAT,
drop_above=1.0):
if chunk_size != 0:
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
dna_rules = FastDNARules()
if asdna:
rules = dna_rules
else:
rules = None
x = RU10Encoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=insert_header,
rules=rules,
error_correction=error_correction,
id_len_format=seed_len_format,
number_of_chunks_len_format=NUMBER_OF_CHUNKS_LEN_FORMAT,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
prepend=prepend,
append=append)
x.prepare()
i = 0
tmp_list = []
while i < while_count:
if seq_seed is not None:
if seq_seed + i >= NUMBER_OF_PACKETS_TO_CREATE:
break
packet = x.create_new_packet(seed=seq_seed + i)
else:
packet = x.create_new_packet()
should_drop_packet(rules, packet)
if packet.error_prob <= drop_above and (
len(tmp_list) < l_size
or packet.error_prob < tmp_list[-1].error_prob):
if packet not in tmp_list:
bisect.insort_left(tmp_list, packet)
else:
elem = next((x for x in tmp_list if x == packet), None)
if packet < elem:
tmp_list.remove(elem)
bisect.insort_left(tmp_list, packet)
if len(tmp_list) > l_size:
tmp_list = tmp_list[:l_size]
i += 1
print([x.error_prob for x in tmp_list])
conf = {
'error_correction': e_correction,
'repair_symbols': _repair_symbols,
'asdna': asdna,
'number_of_splits': _number_of_splits,
'find_minimum_mode': True,
'seq_seed': seq_seed
}
x.save_config_file(conf, section_name="RU10_" + file)
return [ParallelPacket.from_packet(p) for p in tmp_list]
def parse_raw_packet(
self,
packet,
crc_len_format: str = "L",
number_of_chunks_len_format: str = "L",
packet_len_format: str = "I",
id_len_format: str = "L") -> typing.Union[RU10Packet, str]:
"""
Creates a RU10 packet from a raw given packet. Also checks if the packet is corrupted. If any method was used to
create packets from specific chunks, set self.use_method = True. This will treat the last byte of the raw packet
data as the byte that contains the information about the used method ("even", "odd", "window_30 + window" or
"window_40 + window". See RU10Encoder.create_new_packet_from_chunks for further information.
:param packet: A raw packet
:param packet_len_format: Format of the packet length
:param crc_len_format: Format of the crc length
:param number_of_chunks_len_format: Format of the number of chunks length
:param id_len_format: Format of the ID length
:return: RU10Packet or an error message
"""
struct_str = "<" + number_of_chunks_len_format + id_len_format
struct_len = struct.calcsize(struct_str)
""""
if self.error_correction.__code__.co_name == crc32.__code__.co_name:
crc_len = -struct.calcsize("<" + crc_len_format)
payload = packet[:crc_len]
crc = struct.unpack("<" + crc_len_format, packet[crc_len:])[0]
calced_crc = calc_crc(payload)
if crc != calced_crc: # If the Packet is corrupt, try next one
print("[-] CRC-Error - " + str(hex(crc)) + " != " + str(hex(calced_crc)))
self.corrupt += 1
return "CORRUPT"
else:
"""
try:
packet = self.error_correction(packet)
except:
self.corrupt += 1
return "CORRUPT"
data = packet[struct_len:]
if self.use_method:
method_data = bin(data[-1])[2:]
while len(method_data) < 8:
method_data = '0' + method_data
data = data[:-1]
if method_data.startswith('00'):
chunk_lst = [
ch for ch in range(0, self.number_of_chunks + 1)
if ch % 2 == 0
]
elif method_data.startswith('01'):
chunk_lst = [
ch for ch in range(0, self.number_of_chunks + 1)
if ch % 2 != 0
]
elif method_data.startswith('10'):
window = int(method_data[2:], 2)
window_size = 30
start = window * (window_size - 10)
chunk_lst = [
ch for ch in range(start, start + window_size)
if ch <= self.number_of_chunks
]
elif method_data.startswith('11'):
window = int(method_data[2:], 2)
window_size = 40
start = window * (window_size - 10)
chunk_lst = [
ch for ch in range(start, start + window_size)
if ch <= self.number_of_chunks
]
else:
raise RuntimeError(f"Invalid method_data: %s" % method_data)
len_data = struct.unpack(struct_str, packet[0:struct_len])
if self.static_number_of_chunks is None:
self.number_of_chunks = xor_mask(len_data[0],
number_of_chunks_len_format)
unxored_id = xor_mask(len_data[1], id_len_format)
else:
unxored_id = xor_mask(len_data[0], id_len_format)
if self.dist is None:
self.dist = RaptorDistribution(self.number_of_chunks)
_, self.s, self.h = intermediate_symbols(self.number_of_chunks,
self.dist)
if self.correct == 0:
self.createAuxBlocks()
self.correct += 1
if self.use_method:
numbers = choose_packet_numbers(len(chunk_lst),
unxored_id,
self.dist,
systematic=False,
max_l=len(chunk_lst))
used_packets = set([chunk_lst[i] for i in numbers])
else:
used_packets = set(
choose_packet_numbers(self.number_of_chunks,
unxored_id,
self.dist,
systematic=False))
res = RU10Packet(
data,
used_packets,
self.number_of_chunks,
unxored_id,
read_only=True,
packet_len_format=packet_len_format,
crc_len_format=crc_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
id_len_format=id_len_format,
save_number_of_chunks_in_packet=self.static_number_of_chunks is
None)
return res
index.append(key)
data.append(val)
fig, (ax) = plt.subplots(ncols=1)
ax.boxplot(data)
ax.set_xticklabels(index)
plt.show()
plt.plot(num_list)
plt.plot(mean_list)
plt.show()
if __name__ == "__main__":
file = "../.INFILES/Dorn"
chunk_size = 100
norepairsymbols = 6
save_number_of_chunks_in_packet = False
insert_header = False
rules = FastDNARules()
error_correction = lambda x: reed_solomon_encode(x, norepairsymbols)
number_of_chunks = 50
if chunk_size != 0:
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
x = RU10Encoder(file, number_of_chunks, dist, chunk_size=chunk_size, insert_header=insert_header, rules=rules,
error_correction=error_correction, id_len_format="H", number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet)
aa = QualityPacketGen(x)
if __name__ == "__main__":
file = "logo.jpg"
number_of_chunks = 400
insert_header = True
rules = FastDNARules()
norepair_symbols = 4
error_correction_str = "reedsolomon"
error_correction = get_error_correction_encode(error_correction_str, norepair_symbols)
PACKET_LEN_FORMAT = "I"
CRC_LEN_FORMAT = "I"
NUMBER_OF_CHUNKS_LEN_FORMAT = "I"
ID_LEN_FORMAT = "I"
save_number_of_chunks_in_packet = False
upper_bound = 0.9
encoder = RU10Encoder(file, number_of_chunks, RaptorDistribution(number_of_chunks=number_of_chunks),
insert_header=insert_header, pseudo_decoder=None,
chunk_size=0, rules=rules, error_correction=error_correction,
packet_len_format=PACKET_LEN_FORMAT,
crc_len_format=CRC_LEN_FORMAT, number_of_chunks_len_format=NUMBER_OF_CHUNKS_LEN_FORMAT,
id_len_format=ID_LEN_FORMAT, save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
mode_1_bmp=False, prepend="", append="", drop_upper_bound=upper_bound)
encoder.set_overhead_limit(0.40)
# encoder.encode_to_packets()
save_prefix = file + "_" + str(number_of_chunks) + "_" + ("H" if insert_header else "") + "_" + \
("R" if rules is not None else "") + "_" + error_correction_str + "_" + str(norepair_symbols) + \
"_" + PACKET_LEN_FORMAT + "_" + CRC_LEN_FORMAT + "_" + NUMBER_OF_CHUNKS_LEN_FORMAT + "_" + \
ID_LEN_FORMAT + "_" + ("NUMC" if save_number_of_chunks_in_packet else "") + "_" + str(upper_bound)
# plot_chunks_count_in_packets(list(encoder.encodedPackets), save_prefix)
plot_from_csv("logo.jpg_400_H_R_reedsolomon_4_I_I_I_I__0.9_2021-01-20_12-40-52.csv")
def run(seq_seed=None,
file='logo.jpg',
repair_symbols=2,
insert_header=False,
error_correction=reed_solomon_encode,
save_number_of_chunks_in_packet=False,
l_size=1000,
while_count=1000,
chunk_size=0,
number_of_chunks=300,
prepend="",
append="",
seed_len_format=DEFAULT_ID_LEN_FORMAT,
number_of_chunks_len_format=DEFAULT_NUMBER_OF_CHUNKS_LEN_FORMAT,
method='RU10',
mode1bmp=False,
drop_above=0.4,
packets_to_create=None):
global counter
if chunk_size != 0:
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunk_size)
dna_rules = FastDNARules()
if packets_to_create is None:
packets_to_create = math.pow(2, 8 * struct.calcsize(seed_len_format))
rules = dna_rules
if repair_symbols != 0:
dist, error_correction = get_err_dist(method, number_of_chunks,
repair_symbols)
else:
dist = RaptorDistribution(number_of_chunks)
if method == 'RU10':
x = RU10Encoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=insert_header,
rules=rules,
error_correction=error_correction,
id_len_format=seed_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
mode_1_bmp=mode1bmp,
prepend=prepend,
append=append)
x.prepare()
elif method == 'LT':
x = LTEncoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=insert_header,
rules=rules,
error_correction=error_correction,
number_of_chunks_len_format=number_of_chunks_len_format,
id_len_format=seed_len_format,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet)
x.prepareEncoder()
elif method == 'Online':
number_of_chunks = dist.get_size()
x = OnlineEncoder(
file,
number_of_chunks,
dist,
ONLINE_EPS,
ONLINE_QUALITY,
error_correction=error_correction,
quality_len_format="B",
insert_header=False,
check_block_number_len_format=seed_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
rules=rules,
save_number_of_chunks_in_packet=False)
x.prepare()
else:
raise NotImplementedError("Choose: RU10, LT or Online")
i = 0
tmp_list = []
while i < while_count:
if seq_seed is not None:
if seq_seed + i >= packets_to_create:
break
packet = x.create_new_packet(seed=seq_seed + i)
else:
packet = x.create_new_packet()
if i == 0:
print(f"%i , %s" % (len(
packet.get_dna_struct(True)), packet.get_dna_struct(True)))
_ = should_drop_packet(rules, packet)
if packet.error_prob <= drop_above and (
len(tmp_list) < l_size
or packet.error_prob < tmp_list[-1].error_prob):
if packet not in tmp_list:
bisect.insort_left(tmp_list, packet)
else:
elem = next((x for x in tmp_list if x == packet), None)
if packet < elem:
tmp_list.remove(elem)
del elem
bisect.insort_left(tmp_list, packet)
if len(tmp_list) > l_size:
for ele1m in tmp_list[l_size + 1:]:
del ele1m
tmp_list = tmp_list[:l_size]
else:
del packet
i += 1
# += operation is not atomic, so we need to get a lock:
with counter.get_lock():
counter.value += 1
# save_packets_fasta(tmp_list, out_file=method + "_out_partial", file_ending="." + method + "_DNA",
# clear_output=False)
conf = {
'error_correction': error_correction,
'repair_symbols': repair_symbols,
'number_of_splits': _number_of_splits,
'find_minimum_mode': True,
'seq_seed': seq_seed
}
# x.save_config_file(conf, section_name=method + "_" + file)
if x.progress_bar is not None:
x.progress_bar.finish()
return [ParallelPacket.from_packet(p) for p in tmp_list]