def main():
patterns = [
0x61083282abedbf10, 0xcccccccc55555555, 0x1234abcdf5ba03e7,
0x1234abcdf5ba0132, 0
]
random.seed(7)
src = bytearray()
src32 = bytearray()
src3 = bytearray()
for _ in range(512):
src += random.choice(patterns).to_bytes(8, byteorder="big")
for i in range(1024):
#num = (-1 - i) + 2**32
num = 2 * i
src32 += num.to_bytes(4, byteorder="big")
for _ in range(1024):
src3 += random.randint(0, 1024).to_bytes(4, byteorder="big")
src64 = bytearray()
for i in range(512):
#num = (- 1 - i) + 2**64
num = random.randint(1, 10000000)
src64 += num.to_bytes(8, byteorder="big")
src = src64
compressor = WKCompressor(word_size_bytes=4,
dict_size=256,
num_low_bits=10,
debug=False)
wk_compressed = compressor.compress(src)
wk_uncompressed = compressor.decompress(wk_compressed)
wk_huffman_encoded = huffman.compress(wk_compressed)
wk_huffman_decoded = huffman.decompress(wk_huffman_encoded)
wk_huffman_uncompressed = compressor.decompress(wk_huffman_decoded)
lz_compressed = lzma.compress(src)
lz_uncompressed = lzma.decompress(lz_compressed)
zlib_compressed = zlib.compress(src, 9)
zlib_uncompressed = zlib.decompress(zlib_compressed)
bz_compressed = bz2.compress(src)
bz_uncompressed = bz2.decompress(bz_compressed)
print("LRU Histogram: ",
WKCompressor.get_lru_queue_histogram(compressor, wk_compressed))
indices = WKCompressor.get_dict(compressor, wk_compressed)
print(len(indices))
print_results("WK", src, wk_compressed, wk_uncompressed)
print_results("WK Huffman", src, wk_huffman_encoded,
wk_huffman_uncompressed)
print_results("lzma", src, lz_compressed, lz_uncompressed)
print_results("zlib", src, zlib_compressed, zlib_uncompressed)
print_results("bzip2", src, bz_compressed, bz_uncompressed)
def main():
parser = create_parser()
args = parser.parse_args(sys.argv[1:])
output_file_name = args.input_file + ('.dec' if args.extract else '.bin')
input_file = open(args.input_file, 'rb')
output_file = open(output_file_name, 'wb')
outputbytes = b''
if args.extract:
outputbytes = huffman.extract(*huffman.parse_compressed(input_file))
else:
outputbytes = huffman.compress(input_file.read())
output_file.write(outputbytes)
input_file.close()
output_file.close()
def huffman(self,data):
size = len(data)#in bytes
hist = huffman.make_hist(data)
raiz,tree = huffman.make_tree(hist)
bs = 0#0 si es histograma
bc = 4096 # 256*2*8#tiene que ser 16 bits
packHist = self.packet.packHist(self.fs,bs,bc,hist)
self.send(packHist)
#print "PACKETS",len(range(0,size,self.chuckSizeToSend))
for i in range(0,size,self.chuckSizeToSend):
perKb = data[i:i+self.chuckSizeToSend]
com = huffman.compress(tree,perKb)
bs +=1
bc = len(perKb)*8
pack = self.packet.pack(self.fs,bs,bc,com)
self.send(pack)
self.fs += 1
#TEST
"""
def huffman(self, data):
size = len(data) #in bytes
hist = huffman.make_hist(data)
raiz, tree = huffman.make_tree(hist)
bs = 0 #0 si es histograma
bc = 4096 # 256*2*8#tiene que ser 16 bits
packHist = self.packet.packHist(self.fs, bs, bc, hist)
self.send(packHist)
#print "PACKETS",len(range(0,size,self.chuckSizeToSend))
for i in range(0, size, self.chuckSizeToSend):
perKb = data[i:i + self.chuckSizeToSend]
com = huffman.compress(tree, perKb)
bs += 1
bc = len(perKb) * 8
pack = self.packet.pack(self.fs, bs, bc, com)
self.send(pack)
self.fs += 1
#TEST
"""
def compressor(algorithm, wk_compressor=None):
while True:
if not q.empty():
page = q.get()
if page is None:
break
if algorithm == "wk":
compressed = wk_compressor.compress(page)
elif algorithm == "wk-huffman":
wk_compressed = wk_compressor.compress(page)
compressed = huffman.compress(wk_compressed)
elif algorithm == "lzma":
compressed = lzma.compress(page)
elif algorithm == "bzip":
compressed = bz2.compress(page)
else:
print(
"Please enter one of 'wk', 'wk-huffman', 'lzma', or 'bzip'"
)
exit()
compressed_size = len(compressed)
#compressed_size_bytes = struct.pack(">I", compressed_size)
out.write(str(compressed_size) + "\n")
import utility
# 获取数据
# data = utility.get_uniform_data() # 均匀分布
# data = utility.get_normal_data() # 正态分布
data = utility.get_laplace_data() # 拉普拉斯分布
print("原始数据(前300个):")
print(data[0][:300])
(w, h) = data.shape # 取宽 高
data_1D = data.reshape(w * h) # 转为1维
# 压缩
compress_data, average_code_length = huffman.compress(data_1D)
# 计算熵
entropy = utility.entropy(data)
# 计算编码效率
coding_efficiency = utility.coding_efficiency(entropy, average_code_length)
#解码
decompress_data = huffman.decompress(compress_data)
decompress_data = np.array(decompress_data).reshape([w, h])
# print("编码数据：")
# print(compress_data[:100])
print("解码数据（前300个）：")
while True:
time.sleep(1)
except KeyboardInterrupt:
print "stop server"
for thread in threading.enumerate():
if thread.isAlive():
try:
thread._Thread__stop()
except:
print 'No se pudo matar todos los threads'
exit(0)
if __name__ == '__main__' and False:
audio = Audio(byte=True,seconds=1)
while 1:
data = audio.get_chunck()
d = list(data)
print 'size',len(d),'bytes 17.28kb'
hist = huffman.make_hist(d)
raiz,tree = huffman.make_tree(hist)
com = huffman.compress(tree,d)
#print com
d = huffman.decompress(tree,raiz,com)
#print 'salida!',d
t = time.time()
audio.out_chunck(d)
elapse = time.time() - t
print "seconds: %s" % (elapse)
p2.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print "stop server"
for thread in threading.enumerate():
if thread.isAlive():
try:
thread._Thread__stop()
except:
print 'No se pudo matar todos los threads'
exit(0)
if __name__ == '__main__' and False:
audio = Audio(byte=True, seconds=1)
while 1:
data = audio.get_chunck()
d = list(data)
print 'size', len(d), 'bytes 17.28kb'
hist = huffman.make_hist(d)
raiz, tree = huffman.make_tree(hist)
com = huffman.compress(tree, d)
#print com
d = huffman.decompress(tree, raiz, com)
#print 'salida!',d
t = time.time()
audio.out_chunck(d)
elapse = time.time() - t
print "seconds: %s" % (elapse)
Son beau corps a roulé sous la vague marine.
Thétis, les yeux en pleurs, dans le creux d'un rocher
Aux monstres dévorants eut soin de la cacher.
Par ses ordres bientôt les belles Néréides
L'élèvent au-dessus des demeures humides,
Le portent au rivage, et dans ce monument
L'ont, au cap du Zéphir, déposé mollement.
Puis de loin à grands cris appelant leurs compagnes,
Et les Nymphes des bois, des sources, des montagnes,
Toutes frappant leur sein et traînant un long deuil,
Répétèrent : « hélas ! » autour de son cercueil.
Hélas ! chez ton amant tu n'es point ramenée.
Tu n'as point revêtu ta robe d'hyménée.
L'or autour de tes bras n'a point serré de nœuds.
Les doux parfums n'ont point coulé sur tes cheveux."""
text = text.encode("utf-8")
with open("du cote de chez swann.txt", "rb") as f:
text = f.read()
compr = huffman.compress(text)
codelengths = compr["codelengths"]
print(len(text), len(compr["data"]))
decomp = huffman.decompress(compr["data"], codelengths)
assert decomp == text