def get_error_sum(file,
number_of_chunks,
chunk_size,
seq_seed=None,
while_count=1000):
max_seed = np.power(2, 8 * struct.calcsize(SEED_LEN_FORMAT))
dist = RaptorDistribution(number_of_chunks)
dna_rules = FastDNARules()
error_correction = lambda x: reed_solomon_encode(x, NO_REPAIR_SYMBOLS)
encoder = RU10Encoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=INSERT_HEADER,
rules=dna_rules,
error_correction=error_correction,
id_len_format=SEED_LEN_FORMAT,
number_of_chunks_len_format=NUMBER_OF_CHUNKS_LEN_FORMAT,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
prepend="",
append="")
encoder.prepare()
i = 0
res = []
while i < while_count:
if seq_seed is not None:
if seq_seed + i >= max_seed:
break
packet = encoder.create_new_packet(seed=seq_seed + i)
else:
packet = encoder.create_new_packet()
should_drop_packet(dna_rules, packet)
res.append(packet.error_prob)
return res
def encode_to_packets(self) -> float:
self.prepareEncoder()
start: float = time.time()
if self.pseudo_decoder is not None:
while not self.pseudo_decoder.is_decoded():
# This process continues until the receiver signals that the
# message has been received and successfully decoded.
new_pack: Packet = self.create_new_packet()
if self.rules is not None:
while should_drop_packet(self.rules, new_pack,
self.upper_bound):
new_pack = self.create_new_packet()
self.ruleDrop += 1
self.pseudo_decoder.input_new_packet(new_pack)
self.encodedPackets.add(new_pack)
else:
while (len(self.encodedPackets) <
(self.number_of_chunks +
(self.number_of_chunks * self.overhead_limit))
or self.number_of_packets_encoded_already() <
self.number_of_chunks): # 20% Aufschlag
# This process continues until the receiver signals that the
# message has been received and successfully decoded.
pack: Packet = self.create_new_packet()
if self.rules is not None:
while should_drop_packet(self.rules, pack,
self.upper_bound):
pack = self.create_new_packet()
self.ruleDrop += 1
self.encodedPackets.add(pack)
return start
def encode(file, dist_lst, asdna=True, chunk_size=50):
"""
:param file:
:param dist_lst:
:param asdna:
:param chunk_size:
:return:
"""
packets_needed = 0
packets = dict()
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
dist.f = dist_lst
d = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40]
dist.d = d
dna_rules = FastDNARules()
if asdna:
rules = dna_rules
else:
rules = None
x = RU10Encoder(file, number_of_chunks, dist, chunk_size=chunk_size, insert_header=False, rules=rules,
error_correction=nocode, id_len_format="H", number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=False, mode_1_bmp=False)
x.prepare()
y = RU10Decoder.pseudo_decoder(x.number_of_chunks, False)
if y.distribution is None: # self.isPseudo and
y.distribution = RaptorDistribution(x.number_of_chunks)
y.distribution.f = dist_lst
y.distribution.d = d
y.number_of_chunks = x.number_of_chunks
_, y.s, y.h = intermediate_symbols(x.number_of_chunks, y.distribution)
y.createAuxBlocks()
n = 0
for p_tmp in range(45):
packets[p_tmp] = list()
while n < number_of_chunks * 50:
pack = x.create_new_packet()
if packets_needed == 0:
y.input_new_packet(pack)
should_drop_packet(dna_rules, pack)
if pack.get_degree() not in packets:
packets[pack.get_degree()] = list()
packets[pack.get_degree()].append(pack.error_prob)
n += 1
if n >= number_of_chunks and y.is_decoded() and packets_needed == 0:
packets_needed = n
# we dont want to break, we want to generate #chunks * XXX packets!
# break
print("Packets created: " + str(sum([len(x) for x in packets.values()])))
return packets, (packets_needed - number_of_chunks) / 100.0
def blackbox(encoder, decoder):
encoder.encode_to_packets()
encoded_packets = encoder.get_encoded_packets(
) # list(itertools.permutations(..))
i = len(encoded_packets)
invalid_drop = 0
print(bcolors.BOLD + "[+] Created " + str(i) + " Packets" + bcolors.ENDC)
for packet in encoded_packets:
i -= 1
if isinstance(decoder, LTBPDecoder) or isinstance(
decoder, OnlineBPDecoder):
if not should_drop_packet(DNARules, packet):
if decoder.input_new_packet(packet):
decoder.solve()
print(bcolors.OK + "[!] Blackbox dropped " + str(i) +
" random Packets, DNA-Simulator dropped " +
str(invalid_drop) +
" Packets and finished successful" + bcolors.ENDC)
return decoder.is_decoded(), len(encoded_packets), i
else:
invalid_drop += 1
else:
# we are in the FAST_decoder part...
if not should_drop_packet(DNARules, packet):
decoder.input_new_packet(packet)
else:
invalid_drop += 1
# only try to solve if we
if (packet.total_number_of_chunks <= len(encoded_packets) - i -
invalid_drop) and decoder.solve():
print(bcolors.OK + "[!] Blackbox dropped " + str(i) +
" random Packets, DNA-Simulator dropped " +
str(invalid_drop) + " Packets and finished successful" +
bcolors.ENDC)
return decoder.is_decoded(), len(encoded_packets), i
if decoder.GEPP is not None:
decoder.solve()
j = 0 # Make it terminate hard if we doubled our # of encoded Packets...
while (not (decoder.is_decoded()
and decoder.solve())) or j >= 2 * len(encoded_packets):
packet = encoder.create_and_add_new_packet()
if not should_drop_packet(DNARules, packet):
decoder.input_new_packet(packet)
else:
invalid_drop += 1
j += 1
print(bcolors.ITALIC + "[!] Blackbox dropped " + str(i) +
" random Packets." + bcolors.ENDC)
print(bcolors.ITALIC + "[!] DNA-Simulator dropped " + str(invalid_drop) +
" Packets." + bcolors.ENDC)
return decoder.is_decoded(), len(encoder.get_encoded_packets()), i
def create_packets_e_prob(start_num: int, normed_dist: ndarray, number_of_packets: int, rules=None):
dist_obj = RaptorDistribution(__NUM_CHUNKS)
dist_obj.f = normed_dist
dist_obj.d = [x for x in range(0, 41)]
encoder = RU10Encoder(file=__FILE, number_of_chunks=__NUM_CHUNKS, distribution=dist_obj, insert_header=False)
encoder.prepare()
if rules is None:
rules = FastDNARules()
packets_e_prob = []
for i in range(start_num, start_num + number_of_packets):
packet = encoder.create_new_packet(seed=i)
should_drop_packet(rules, packet)
packets_e_prob.append(packet.error_prob)
del packet
del encoder
return packets_e_prob
def encode(self, file, asdna=True, error_correction=nocode, insert_header=False,
save_number_of_chunks_in_packet=False, mode_1_bmp=False, chunk_size=50):
packets_needed = 0
packets = dict()
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
dist.f = self.X
dist.d = self.d
dna_rules = FastDNARules()
if asdna:
rules = dna_rules
else:
rules = None
x = RU10Encoder(file, number_of_chunks, dist, chunk_size=chunk_size, insert_header=insert_header, rules=rules,
error_correction=error_correction, id_len_format="H", number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet, mode_1_bmp=mode_1_bmp)
x.prepare()
y = RU10Decoder.pseudo_decoder(x.number_of_chunks, False)
if y.distribution is None: # self.isPseudo and
y.distribution = RaptorDistribution(x.number_of_chunks)
y.distribution.f = self.X
y.distribution.d = self.d
y.number_of_chunks = x.number_of_chunks
_, y.s, y.h = intermediate_symbols(x.number_of_chunks, y.distribution)
y.createAuxBlocks()
n = 0
for p_tmp in range(45):
packets[p_tmp] = list()
while n < number_of_chunks * 50:
pack = x.create_new_packet()
if packets_needed == 0:
y.input_new_packet(pack)
should_drop_packet(dna_rules, pack)
if pack.get_degree() not in packets:
packets[pack.get_degree()] = list()
packets[pack.get_degree()].append(pack.error_prob)
n += 1
if n >= number_of_chunks and y.is_decoded() and packets_needed == 0:
packets_needed = n
# we dont want to break, we want to generate #chunks * XXX packets!
# break
print("Packets created: " + str(sum([len(x) for x in packets.values()])))
return packets, (packets_needed - number_of_chunks) / 100.0
def _encode(pseudo):
pack = self.create_new_packet()
if self.rules is not None:
while should_drop_packet(self.rules, pack, self.upper_bound):
del pack
pack = self.create_new_packet()
self.ruleDrop += 1
self.encodedPackets.add(pack)
if pseudo:
self.pseudo_decoder.input_new_packet(pack)
self.update_progress_bar()
def encode(file, chunk_size, dist, as_dna=True, repeats=15):
"""
Encodes the file to packets until the pseudo decoder was able to decode it 'repeats' times with the given chunk size
and the distribution list.
:param file: File to encode.
:param chunk_size: Chunksize to use.
:param dist: The distribution to calculate the average error and overhead for.
:param as_dna: If true uses the DNA Rules.
:param repeats: Number of En-/Decoding cycles.
:return:
"""
degree_dict = {}
overhead_lst = []
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(file, chunk_size, insert_header=False)
distribution = RaptorDistribution(number_of_chunks)
distribution.f = dist
distribution.d = [x for x in range(0, 41)]
if as_dna:
rules = FastDNARules()
else:
rules = None
encoder = RU10Encoder(file, number_of_chunks, distribution, insert_header=False, rules=rules,
error_correction=nocode, id_len_format="H", number_of_chunks_len_format="B",
save_number_of_chunks_in_packet=False, mode_1_bmp=False)
encoder.prepare()
for _ in range(0, repeats):
encoder.random_state = np.random.RandomState()
# print("Master-Seed used: " + str(encoder.random_state.get_state()[1][0]))
pseudo_decoder = create_pseudo_decoder(encoder.number_of_chunks, distribution)
needed_packets = 0
while pseudo_decoder.GEPP is None or not pseudo_decoder.is_decoded():
needed_packets += 1
packet = encoder.create_new_packet()
pseudo_decoder.input_new_packet(packet)
should_drop_packet(rules, packet)
if packet.get_degree() not in degree_dict:
degree_dict[packet.get_degree()] = list()
degree_dict[packet.get_degree()].append(min(packet.error_prob, 1.0))
overhead = (needed_packets - encoder.number_of_chunks) / 100.0
overhead_lst.append(overhead)
return sum(overhead_lst) / len(overhead_lst), degree_dict
def __init__(self, encoder):
encoder.prepare()
i = 0
tmp_list = []
while i < 1000:
packet = encoder.create_new_packet()
p_res = should_drop_packet(rules, packet)
tmp_list.append(packet)
i += 1
QualityAnalyzer(tmp_list)
def run(seq_seed=None,
file='logo.jpg',
asdna=True,
insert_header=False,
error_correction=reed_solomon_encode,
save_number_of_chunks_in_packet=False,
l_size=1000,
while_count=1000,
chunk_size=0,
number_of_chunks=300,
prepend="",
append="",
seed_len_format=ID_LEN_FORMAT,
drop_above=1.0):
if chunk_size != 0:
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunk_size)
dist = RaptorDistribution(number_of_chunks)
dna_rules = FastDNARules()
if asdna:
rules = dna_rules
else:
rules = None
x = RU10Encoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=insert_header,
rules=rules,
error_correction=error_correction,
id_len_format=seed_len_format,
number_of_chunks_len_format=NUMBER_OF_CHUNKS_LEN_FORMAT,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
prepend=prepend,
append=append)
x.prepare()
i = 0
tmp_list = []
while i < while_count:
if seq_seed is not None:
if seq_seed + i >= NUMBER_OF_PACKETS_TO_CREATE:
break
packet = x.create_new_packet(seed=seq_seed + i)
else:
packet = x.create_new_packet()
should_drop_packet(rules, packet)
if packet.error_prob <= drop_above and (
len(tmp_list) < l_size
or packet.error_prob < tmp_list[-1].error_prob):
if packet not in tmp_list:
bisect.insort_left(tmp_list, packet)
else:
elem = next((x for x in tmp_list if x == packet), None)
if packet < elem:
tmp_list.remove(elem)
bisect.insort_left(tmp_list, packet)
if len(tmp_list) > l_size:
tmp_list = tmp_list[:l_size]
i += 1
print([x.error_prob for x in tmp_list])
conf = {
'error_correction': e_correction,
'repair_symbols': _repair_symbols,
'asdna': asdna,
'number_of_splits': _number_of_splits,
'find_minimum_mode': True,
'seq_seed': seq_seed
}
x.save_config_file(conf, section_name="RU10_" + file)
return [ParallelPacket.from_packet(p) for p in tmp_list]
def run(seq_seed=None,
file='logo.jpg',
repair_symbols=2,
insert_header=False,
error_correction=reed_solomon_encode,
save_number_of_chunks_in_packet=False,
l_size=1000,
while_count=1000,
chunk_size=0,
number_of_chunks=300,
prepend="",
append="",
seed_len_format=DEFAULT_ID_LEN_FORMAT,
number_of_chunks_len_format=DEFAULT_NUMBER_OF_CHUNKS_LEN_FORMAT,
method='RU10',
mode1bmp=False,
drop_above=0.4,
packets_to_create=None):
global counter
if chunk_size != 0:
number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size(
file, chunk_size)
dna_rules = FastDNARules()
if packets_to_create is None:
packets_to_create = math.pow(2, 8 * struct.calcsize(seed_len_format))
rules = dna_rules
if repair_symbols != 0:
dist, error_correction = get_err_dist(method, number_of_chunks,
repair_symbols)
else:
dist = RaptorDistribution(number_of_chunks)
if method == 'RU10':
x = RU10Encoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=insert_header,
rules=rules,
error_correction=error_correction,
id_len_format=seed_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet,
mode_1_bmp=mode1bmp,
prepend=prepend,
append=append)
x.prepare()
elif method == 'LT':
x = LTEncoder(
file,
number_of_chunks,
dist,
chunk_size=chunk_size,
insert_header=insert_header,
rules=rules,
error_correction=error_correction,
number_of_chunks_len_format=number_of_chunks_len_format,
id_len_format=seed_len_format,
save_number_of_chunks_in_packet=save_number_of_chunks_in_packet)
x.prepareEncoder()
elif method == 'Online':
number_of_chunks = dist.get_size()
x = OnlineEncoder(
file,
number_of_chunks,
dist,
ONLINE_EPS,
ONLINE_QUALITY,
error_correction=error_correction,
quality_len_format="B",
insert_header=False,
check_block_number_len_format=seed_len_format,
number_of_chunks_len_format=number_of_chunks_len_format,
rules=rules,
save_number_of_chunks_in_packet=False)
x.prepare()
else:
raise NotImplementedError("Choose: RU10, LT or Online")
i = 0
tmp_list = []
while i < while_count:
if seq_seed is not None:
if seq_seed + i >= packets_to_create:
break
packet = x.create_new_packet(seed=seq_seed + i)
else:
packet = x.create_new_packet()
if i == 0:
print(f"%i , %s" % (len(
packet.get_dna_struct(True)), packet.get_dna_struct(True)))
_ = should_drop_packet(rules, packet)
if packet.error_prob <= drop_above and (
len(tmp_list) < l_size
or packet.error_prob < tmp_list[-1].error_prob):
if packet not in tmp_list:
bisect.insort_left(tmp_list, packet)
else:
elem = next((x for x in tmp_list if x == packet), None)
if packet < elem:
tmp_list.remove(elem)
del elem
bisect.insort_left(tmp_list, packet)
if len(tmp_list) > l_size:
for ele1m in tmp_list[l_size + 1:]:
del ele1m
tmp_list = tmp_list[:l_size]
else:
del packet
i += 1
# += operation is not atomic, so we need to get a lock:
with counter.get_lock():
counter.value += 1
# save_packets_fasta(tmp_list, out_file=method + "_out_partial", file_ending="." + method + "_DNA",
# clear_output=False)
conf = {
'error_correction': error_correction,
'repair_symbols': repair_symbols,
'number_of_splits': _number_of_splits,
'find_minimum_mode': True,
'seq_seed': seq_seed
}
# x.save_config_file(conf, section_name=method + "_" + file)
if x.progress_bar is not None:
x.progress_bar.finish()
return [ParallelPacket.from_packet(p) for p in tmp_list]