def blackboxTest(file, number_of_chunks=800, droprate=0.02, seed=2, overhead=0.05): print("#### Starting Blackbox Test with " + str(number_of_chunks) + " Chunks and a droprate of " + str(droprate) + " ####") start = time.time() dist = RobustSolitonDistribution(S=number_of_chunks, seed=seed) # dist = IdealSolitonDistribution(S=number_of_chunks, seed=seed) # pseudo = LTBPDecoder.pseudo_decoder(number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist) # , pseudo_decoder=pseudo) decoder = LTDecoder.pseudo_decoder(number_of_chunks, read_all_before_decode=True) encoder.set_overhead_limit(overhead) result, numberOfEncodedPackets, dropedCount, solvedCount = blackbox( encoder, decoder, droprate=droprate) end = time.time() - start print("Blackbox-Decode " + ("successful" if result else "NOT successful") + " after " + str(round(end, 4)) + " sec.") return [ dist.get_config_string(), result, numberOfEncodedPackets, dropedCount, solvedCount, round(end, 4), number_of_chunks, ]
def blackboxLTTest(file, number_of_chunks=800, seed=2, chunk_size=0, overhead=0.20, scale=1.0): print(bcolors.OK + "Starting Blackbox Test with " + str(number_of_chunks) + " Chunks" + bcolors.ENDC) start = time.time() dist = RobustSolitonDistribution(S=number_of_chunks, seed=seed) algo_type.clear() algo_type.append("LT_" + str(number_of_chunks) + "_" + str(dist.get_config_string())) encoder = LTEncoder(file, number_of_chunks, dist, chunk_size=chunk_size) encoder.set_overhead_limit(overhead) decoder = LTDecoder.pseudo_decoder(number_of_chunks) decoder.set_read_all_before_decode(True) result, dec_input, invalid_drop = blackbox(encoder, decoder, scale=scale) end = time.time() - start print(bcolors.BLUE + "Blackbox-Decode " + (bcolors.OK + "successful" if result else bcolors.ERR + "NOT successful") + bcolors.END + bcolors.BLUE + " after " + str(round(end, 4)) + " sec." + bcolors.ENDC) return [ dist.get_config_string(), result, number_of_chunks, dec_input, invalid_drop, round(end, 4), ]
def decode(file, error_correction=nocode, null_is_terminator=False, mode_1_bmp=False, number_of_chunks=STATIC_NUM_CHUNKS, use_header_chunk=False, id_len_format=ID_LEN_FORMAT, number_of_chunks_len_format=NUMBER_OF_CHUNKS_LEN_FORMAT, packet_len_format=PACKET_LEN_FORMAT, crc_len_format=CRC_LEN_FORMAT, read_all=READ_ALL_BEFORE_DECODER, distribution_cfg_str=""): dist = ErlichZielinskiRobustSolitonDistribution(number_of_chunks, seed=2) if distribution_cfg_str != "": # parse distribution_cfg_str and create distribution with the defined settings... splt = distribution_cfg_str.split("_") mode = splt[1] if mode == "ErlichZielinskiRobustSoliton": k = int(splt[2].split("=")[1]) delta = float(splt[3].split("=")[1]) c = float(splt[4].split("=")[1]) dist = ErlichZielinskiRobustSolitonDistribution(k, delta, c, seed=2) elif mode == "RobustSoliton": k = int(splt[2].split("=")[1]) delta = float(splt[3].split("=")[1]) dist = RobustSolitonDistribution(number_of_chunks, k, delta, seed=2) elif mode == "IdealSoliton": dist = IdealSolitonDistribution(number_of_chunks, seed=2) """try: decoder = LTBPDecoder(file, error_correction=error_correction, use_headerchunk=HEADER_CHUNK, static_number_of_chunks=STATIC_NUM_CHUNKS, implicit_mode=IMPLICIT_MODE, dist=dist) print("[1/2] Approximation Decode") _internal(decoder) if not decoder.is_decoded(): print("[2/2] Approximation Decode") _internal(decoder) decoder.saveDecodedFile(null_is_terminator=NULL_IS_TERMINATOR) except Exception as e:""" print("[X/2] Falling back to Gauss-Mode") print("Falling back to Gauss-Mode") decoder = LTDecoder(file, error_correction=error_correction, use_headerchunk=use_header_chunk, static_number_of_chunks=number_of_chunks, implicit_mode=IMPLICIT_MODE, dist=dist) decoder.read_all_before_decode = read_all decoder.decode(number_of_chunks_len_format=number_of_chunks_len_format, seed_len_format=id_len_format, degree_len_format="H") decoder.solve() decoder.saveDecodedFile(null_is_terminator=null_is_terminator, print_to_output=PRINT_TO_OUTPUT)
def vergleich(): S = 50 for a in ["log", ""]: delt = 0.1 robust = RobustSolitonDistribution(S=S, K=8, delta=delt, seed=0) print(robust.pre_comp_dist) if a == "log": plt.semilogy([0.0] + robust.pre_comp_dist, label="K=8, $\delta$ = 0.1") else: plt.plot([0.0] + robust.pre_comp_dist, label="K=8, $\delta$ = 0.1") robust = RobustSolitonDistribution(S=S, K=8, delta=1.0, seed=0) print(robust.pre_comp_dist) if a == "log": plt.semilogy([0.0] + robust.pre_comp_dist, label="K=8, $\delta$ = 1.0") else: plt.plot([0.0] + robust.pre_comp_dist, label="K=8, $\delta$ = 1.0") robust = RobustSolitonDistribution(S=S, K=15, delta=0.5, seed=0) print(robust.pre_comp_dist) if a == "log": plt.semilogy([0.0] + robust.pre_comp_dist, label="K=15, $\delta$ = 0.5") else: plt.plot([0.0] + robust.pre_comp_dist, label="K=15, $\delta$ = 0.5") plt.ylabel("Probability") plt.xlabel("Degree") manager = plt.get_current_fig_manager() # manager.resize(*manager.window.maxsize()) plt.grid(True) plt.tight_layout() plt.legend() plt.show(block=False) plt.savefig( "../plotDists/Vergleich_RobustSolitonS50_K8_delta" + str(delt) + "_" + a + ".pdf", bbox_inches="tight", ) plt.savefig("../plotDists/Vergleich_RobustSolitonS50_K8_delta" + str(delt) + "_" + a + ".svg") plt.close()
def blackboxLTTest(file, number_of_chunks=800, seed=2, chunk_size=0): print("Starting Blackbox Test with " + str(number_of_chunks) + " Chunks") start = time.time() dist = RobustSolitonDistribution(S=number_of_chunks, seed=seed) pseudo = LTBPDecoder.pseudo_decoder(number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo, chunk_size=chunk_size) decoder = LTDecoder.pseudo_decoder(number_of_chunks) result, numberOfEncodedPackets, dropedCount = blackbox(encoder, decoder) end = time.time() - start print("Blackbox-Decode " + ("successful" if result else "NOT successful") + " after " + str(round(end, 4)) + " sec.") return [ dist.get_config_string(), result, numberOfEncodedPackets, dropedCount, round(end, 4), number_of_chunks, ]
def main(): S = 50 for a in ["log", ""]: for delt in np.arange(0.1, 1.1, 0.1): robust = RobustSolitonDistribution(S=S, K=8, delta=delt, seed=0) print(robust.pre_comp_dist) if a == "log": plt.semilogy([0.0] + robust.pre_comp_dist) else: plt.plot([0.0] + robust.pre_comp_dist) plt.ylabel("Probability") plt.xlabel("Degree") manager = plt.get_current_fig_manager() # manager.resize(*manager.window.maxsize()) plt.grid(True) plt.tight_layout() plt.show(block=False) plt.savefig( "../plotDists/RobustSolitonS50_K8_delta" + str(delt) + "_" + a + ".pdf", bbox_inches="tight", ) plt.savefig("../plotDists/RobustSolitonS50_K8_delta" + str(delt) + "_" + a + ".svg") plt.close() S = 50 for a in ["log", ""]: robust = IdealSolitonDistribution(S=S, seed=0) print(robust.pre_comp_dist) if a == "log": plt.semilogy([0.0] + robust.pre_comp_dist) else: plt.plot([0.0] + robust.pre_comp_dist) plt.ylabel("Probability") plt.xlabel("Degree") manager = plt.get_current_fig_manager() manager.resize(*manager.window.maxsize()) plt.grid(True) plt.tight_layout() plt.show(block=False) plt.savefig("../plotDists/IdealSolitonS50_" + a + ".pdf", bbox_inches="tight") plt.savefig("../plotDists/IdealSolitonS50_" + a + ".svg") plt.close()
def encode(file, error_correction=nocode, insert_header=INSERT_HEADER, save_number_of_chunks=NUMBER_OF_CHUNKS_IN_PACKET, save_as_fasta=True, save_as_zip=True, overhead=5.0, upper_bound=1.0): number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size( file, chunk_size=CHUNK_SIZE, insert_header=insert_header) print("Number of Chunks=%s" % number_of_chunks) #dist = ErlichZielinskiRobustSolitonDistribution(number_of_chunks, seed=2) #dist = IdealSolitonDistribution(number_of_chunks, seed=2) dist = RobustSolitonDistribution(number_of_chunks, seed=2) encoder = LTEncoder( file, number_of_chunks, dist, insert_header=insert_header, rules=DNARules_ErlichZielinski(), error_correction=error_correction, number_of_chunks_len_format="H", id_len_format="H", used_packets_len_format="H", save_number_of_chunks_in_packet=save_number_of_chunks, implicit_mode=IMPLICIT_MODE, drop_upper_bound=upper_bound) encoder.set_overhead_limit(overhead) encoder.encode_to_packets() if save_as_fasta: encoder.save_packets_fasta(file_ending="_LT", seed_is_filename=True) elif save_as_zip: encoder.save_packets_zip(save_as_dna=True, file_ending="_LT", seed_is_filename=True) else: encoder.save_packets(True, save_as_dna=True, seed_is_filename=True, clear_output=True) encoder.save_packets(split_to_multiple_files=True, save_as_dna=True) print("Number of Chunks=%s" % encoder.number_of_chunks) return encoder
def main(): file = "../.INFILES/b_lq.webm" for number_of_chunks in [500, 600, 700, 800, 900, 1000, 1100, 1200]: # for epsilon in [0.03]: for overhead in np.arange(0.05, 0.50, 0.01): for _ in range(3): dist = RobustSolitonDistribution(S=number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist) encoder.set_overhead_limit(overhead) encoder.encode_to_packets() aprrox_decoder = LTBPDecoder(None) start = time.time() for packet in encoder.encodedPackets: aprrox_decoder.input_new_packet(packet) aprrox_decoder.solve() end = time.time() - start print("Approx.," + file + "," + str(number_of_chunks) + "," + str(len(encoder.encodedPackets)) + "," + str(aprrox_decoder.is_decoded()) + "," + str(end))
def main(file): print("###### LT Codec number_of_chunks = 700 ######") print("##### Ideal Soliton Distribution #####") print("### LT without prioritized Packets ###") start = time.time() number_of_chunks = 700 dist = IdealSolitonDistribution(number_of_chunks, seed=2) encoder = LTEncoder(file, number_of_chunks, dist) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### LT without prioritized Packets with pseudo_decoder ###") start = time.time() number_of_chunks = 700 dist = IdealSolitonDistribution(number_of_chunks, seed=2) pseudo = LTBPDecoder.pseudo_decoder(number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### LT with prioritized Packets ###") start = time.time() number_of_chunks = 700 dist = IdealSolitonDistribution(number_of_chunks, seed=2) encoder = LTEncoder(file, number_of_chunks, dist, prioritized_packets=[1, 3, 5]) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### LT with prioritized Packets AND pseudo_decoder ###") start = time.time() number_of_chunks = 700 dist = IdealSolitonDistribution(number_of_chunks, seed=2) pseudo = LTBPDecoder.pseudo_decoder(number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo, prioritized_packets=[1, 3, 5]) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### LT with prioritized Packets AND Gauss-pseudo_decoder ###") start = time.time() number_of_chunks = 700 dist = IdealSolitonDistribution(number_of_chunks, seed=2) pseudo = LTDecoder.pseudo_decoder(number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo, prioritized_packets=[1, 3, 5]) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("##### Robust Soliton Distribution #####") print("### LT without prioritized Packets ###") start = time.time() number_of_chunks = 700 dist = RobustSolitonDistribution(number_of_chunks, seed=2) encoder = LTEncoder(file, number_of_chunks, dist) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### LT without prioritized Packets with Gauss-pseudo_decoder ###") start = time.time() number_of_chunks = 700 dist = RobustSolitonDistribution(number_of_chunks, seed=2) pseudo = LTDecoder.pseudo_decoder(number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### LT with prioritized Packets ###") start = time.time() number_of_chunks = 700 dist = RobustSolitonDistribution(number_of_chunks, seed=2) encoder = LTEncoder(file, number_of_chunks, dist, prioritized_packets=[1, 3, 5]) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### LT with prioritized Packets AND Gauss-pseudo_decoder ###") start = time.time() number_of_chunks = 700 dist = RobustSolitonDistribution(number_of_chunks, seed=2) pseudo = LTDecoder.pseudo_decoder(number_of_chunks) encoder = LTEncoder(file, number_of_chunks, dist, pseudo_decoder=pseudo, prioritized_packets=[1, 3, 5]) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print( "###### Online Codec - number_of_chunks = 2500, eps=0.01, quality=3 ######" ) print("### Online without pseudo_decoder ###") start = time.time() number_of_chunks = 2500 epsilon = 0.01 quality = 3 dist = OnlineDistribution(epsilon) # infer number_of_chunks form Distribution: number_of_chunks = dist.get_size() pseudo = OnlineBPDecoder.pseudo_decoder() encoder = OnlineEncoder(file, number_of_chunks, dist, epsilon, quality) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n") print("### Online with pseudo_decoder ###") start = time.time() number_of_chunks = 2500 epsilon = 0.01 quality = 3 dist = OnlineDistribution(epsilon) # infer number_of_chunks form Distribution: number_of_chunks = dist.get_size() pseudo = OnlineBPDecoder.pseudo_decoder() encoder = OnlineEncoder(file, number_of_chunks, dist, epsilon, quality, pseudo_decoder=pseudo) encoder.encode_to_packets() end = time.time() - start print("Finished encoding after " + str(round(end, 4)) + " sec. " + str(len(encoder.get_encoded_packets())) + " Packets encoded.\n")
def test_suite(as_dna, decoder_instance, distribution, use_header, implicit_mode): dir_path = os.getcwd() try: os.remove(dir_path + "/" + file) except: print("Not deleting, File did not exists") shutil.copyfile(dir_path + "/" + cmp_file, dir_path + "/" + file) chunksize = 200 number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size( file, chunksize) pseudo_decoder = decoder_instance.pseudo_decoder(number_of_chunks) if distribution == "robust": dist = RobustSolitonDistribution(S=number_of_chunks, delta=0.2, seed=2) elif distribution == "ideal": dist = IdealSolitonDistribution(S=number_of_chunks, seed=2) else: dist = ErlichZielinskiRobustSolitonDistribution(k=number_of_chunks, delta=0.2, seed=2) rules = FastDNARules() if as_dna else None encoder = LTEncoder(file, number_of_chunks, dist, chunk_size=chunksize, pseudo_decoder=pseudo_decoder, rules=rules, insert_header=use_header, number_of_chunks_len_format="H", id_len_format="H", used_packets_len_format="H", implicit_mode=implicit_mode) encoder.encode_to_packets() encoder.save_packets(split_to_multiple_files=True, save_as_dna=as_dna) assert pseudo_decoder.is_decoded() and pseudo_decoder.getSolvedCount( ) == encoder.number_of_chunks assert os.path.exists(out_dir) decoder = decoder_instance(out_dir, use_headerchunk=use_header, dist=dist, implicit_mode=implicit_mode) decoder.decodeFolder(number_of_chunks_len_format="H", seed_len_format="H", degree_len_format="H") assert decoder.is_decoded() and decoder.getSolvedCount( ) == encoder.number_of_chunks os.remove(file) decoder.saveDecodedFile(print_to_output=False) if not use_header: out_file = "DEC_LT_" + file else: out_file = file assert os.path.exists(out_file) and filecmp.cmp(out_file, cmp_file) if decoder_instance == LTBPDecoder: # since ApproxDecoder defines an upper bound Gauss-Decoder MUST be able to decode! decoder = LTDecoder(out_dir, use_headerchunk=use_header, dist=dist, implicit_mode=implicit_mode) decoder.decodeFolder(number_of_chunks_len_format="H", seed_len_format="H", degree_len_format="H") assert (decoder.is_decoded() and decoder.getSolvedCount() == encoder.number_of_chunks) os.remove(out_file) decoder.saveDecodedFile(print_to_output=False) assert os.path.exists(out_file) and filecmp.cmp(out_file, cmp_file) shutil.rmtree(out_dir)
def test_size_shrink(): dir_path = os.getcwd() try: os.remove(dir_path + "/" + file) except: print("Not deleting, File did not exists") shutil.copyfile(dir_path + "/" + cmp_file, dir_path + "/" + file) chunksize = 200 number_of_chunks = Encoder.get_number_of_chunks_for_file_with_chunk_size( file, chunksize) dist = RobustSolitonDistribution(S=number_of_chunks, delta=0.2, seed=2) encoder1 = LTEncoder(file, number_of_chunks, dist, chunk_size=chunksize, rules=None, insert_header=False, number_of_chunks_len_format="H", id_len_format="H", used_packets_len_format="H", implicit_mode=False) number_of_chunks = encoder1.number_of_chunks encoder1.prepareEncoder() packet1 = encoder1.create_new_packet() encoder2 = LTEncoder(file, number_of_chunks, dist, chunk_size=chunksize, rules=None, insert_header=False, number_of_chunks_len_format="H", id_len_format="H", used_packets_len_format="H", implicit_mode=True) encoder2.prepareEncoder() packet2 = encoder2.create_new_packet() assert len(packet1.get_dna_struct(True)) > len( packet2.get_dna_struct(True)) encoder3 = LTEncoder(file, number_of_chunks, dist, chunk_size=chunksize, rules=None, insert_header=False, number_of_chunks_len_format="I", id_len_format="I", used_packets_len_format="I", implicit_mode=False) encoder3.prepareEncoder() packet3 = encoder3.create_new_packet() encoder4 = LTEncoder(file, number_of_chunks, dist, chunk_size=chunksize, rules=None, insert_header=False, number_of_chunks_len_format="I", id_len_format="I", used_packets_len_format="I", implicit_mode=True) encoder4.prepareEncoder() packet4 = encoder4.create_new_packet() assert len(packet3.get_dna_struct(True)) > len( packet4.get_dna_struct(True)) assert len(packet3.get_dna_struct(True)) > len( packet1.get_dna_struct(True)) assert len(packet4.get_dna_struct(True)) > len( packet2.get_dna_struct(True))