def test_bmp_file_2(self):
file_orig = "./testdata/additional/bmp/poirot_new.bmp"
compress(file_orig, FILE_COMPRESSED_NAME, 1024 * 1024, False)
decompress(FILE_COMPRESSED_NAME, FILE_DECOMPRESSED_NAME)
self.assertTrue(compare_files(file_orig, FILE_DECOMPRESSED_NAME))
os.remove(FILE_COMPRESSED_NAME)
os.remove(FILE_DECOMPRESSED_NAME)
def test_pdf_file(self):
file_orig = "./testdata/additional/pdf/Izb_04_08.pdf"
compress(file_orig, FILE_COMPRESSED_NAME, 1024 * 1024, False)
decompress(FILE_COMPRESSED_NAME, FILE_DECOMPRESSED_NAME)
self.assertTrue(compare_files(file_orig, FILE_DECOMPRESSED_NAME))
os.remove(FILE_COMPRESSED_NAME)
os.remove(FILE_DECOMPRESSED_NAME)
def test_122kb_file_16mb_dict(self):
file_orig = "./testdata/basic/KB0122"
compress(file_orig, FILE_COMPRESSED_NAME, 3500, True)
decompress(FILE_COMPRESSED_NAME, FILE_DECOMPRESSED_NAME)
self.assertTrue(compare_files(file_orig, FILE_DECOMPRESSED_NAME))
os.remove(FILE_COMPRESSED_NAME)
os.remove(FILE_DECOMPRESSED_NAME)
def test_4kb_file(self):
file_orig = "./testdata/basic/KB0004"
compress(file_orig, FILE_COMPRESSED_NAME, 1000, False)
decompress(FILE_COMPRESSED_NAME, FILE_DECOMPRESSED_NAME)
self.assertTrue(compare_files(file_orig, FILE_DECOMPRESSED_NAME))
os.remove(FILE_COMPRESSED_NAME)
os.remove(FILE_DECOMPRESSED_NAME)
def main():
pg.mixer.music.load('sons/moon-light.mp3')
pg.mixer.music.play(0)
screen = v.tela
d.decompress('imagens/menu')
logo = pg.transform.scale(pg.image.load('imagens/logo.png'),
(v.largura, v.altura))
cps = pg.transform.scale(pg.image.load('imagens/cps.png'),
(v.largura, v.altura))
kafka = pg.transform.scale(pg.image.load('imagens/kafka.png'),
(v.largura, v.altura))
background = pg.transform.scale(pg.image.load('imagens/menu.jpg'),
(v.largura, v.altura))
screen.blit(cps, (0, 0))
pg.display.update()
sleep(2)
screen.blit(kafka, (0, 0))
pg.display.update()
sleep(2)
screen.blit(logo, (0, 0))
pg.display.update()
sleep(2)
screen.blit(background, (0, 0))
myfont = pg.font.SysFont("monospace", 15)
label = myfont.render("Select the language / Selecione o idioma", 1,
(255, 255, 0))
screen.blit(label, (500, 275))
escolha = 0
while escolha == 0:
eua = pg.transform.scale(pg.image.load('imagens/eua.png'), (50, 40))
screen.blit(eua, (600, 300))
br = pg.transform.scale(pg.image.load('imagens/bandeira_nacional.png'),
(50, 40))
screen.blit(br, (700, 300))
recteua = eua.get_rect(topleft=(600, 300))
rectbr = br.get_rect(topleft=(700, 300))
for evento in pg.event.get():
pos = pg.mouse.get_pos()
if evento.type == QUIT:
os.remove('imagens/menu.jpg')
os.remove('imagens/sala.jpg')
os.remove('imagens/menu.jpg')
pg.quit()
elif evento.type == KEYDOWN:
if evento.key == K_ESCAPE:
os.remove('imagens/menu.jpg')
os.remove('imagens/sala.jpg')
os.remove('imagens/menu.jpg')
pg.quit()
elif evento.type == pg.MOUSEBUTTONDOWN:
if pg.mouse.get_pressed()[0] and recteua.collidepoint(pos):
escolha = 1
elif pg.mouse.get_pressed()[0] and rectbr.collidepoint(pos):
escolha = 2
pg.display.update()
if escolha == 1:
return Menu.main("en")
if escolha == 2:
return Menu.main("br")
def testfile(filename, cond_huffman):
print '\n' + '%' * 90
report = {'name' : filename}
start_time = time()
report['before_size'] = str(os.path.getsize(filename)) + " bytes"
# Compress
compress(filename, [cond_huffman == 'True'])
report['comp_time'] = '%.2f' % (time() - start_time) + 's'
report['after_size'] = str(os.path.getsize(filename + '.z')) + " bytes"
report['ratio'] = Fore.YELLOW + str(os.path.getsize(filename + '.z') * 100 /
os.path.getsize(filename)) + '%' + Fore.RESET
start_time = time()
print '-' * 60
# Decompress
decompress(filename + '.z', filename + '.new')
report['decomp_time'] = '%.2f' % (time() - start_time) + 's'
success = True
path = os.path.dirname(filename) + '/'
if filecmp.cmp(path + 'out2', path + 'out22'):
print Fore.GREEN + 'Sequences files are identical' + Fore.RESET
else:
print Fore.RED + 'Sequences files are different!' + Fore.RESET
success = False
if filecmp.cmp(path + 'out3', path + 'out33'):
print Fore.GREEN + 'Quality files are identical' + Fore.RESET
else:
print Fore.RED + 'Quality files are different!' + Fore.RESET
success = False
if filecmp.cmp(path + 'out1', path + 'out11'):
print Fore.GREEN + 'Info files are identical' + Fore.RESET
else:
print Fore.RED + 'Info files are different!' + Fore.RESET
success = False
if filecmp.cmp(filename, filename + '.new'):
print Fore.GREEN + 'Files are identical' + Fore.RESET
else:
print Fore.RED + 'Files are different!' + Fore.RESET
success = False
report['success'] = success
return report
def test_compressAndDecompress(self):
inputBuffer = StringIO()
outputBuffer = StringIO()
multiFastaData = '>Test \nAACCTGACT\n>Test2 \nAACCTGAA'
inputBuffer.write(multiFastaData) # Sample multifasta
inputBuffer.seek(0)
byteBuffer = BytesIO() # Init a byte buffer to write to
compress.compress(inputBuffer, byteBuffer) # Compress to bytebuffer
byteBuffer.seek(0) # Reset the pointer
decompress.decompress(byteBuffer, outputBuffer) # Decompress from bytebuffer
outputBuffer.seek(0)
body = outputBuffer.read()
self.assertEqual(body.strip(), multiFastaData.strip(), 'Full cycle should be lossless') # Newlines can be added in decompression
def main(argv):
operation, input_file, output_file = get_startup_arguments(argv)
if operation == '-c':
if output_file:
file_name = output_file
else:
file_name = input_file.replace('.txt', '.z78')
compress(input_file, file_name)
elif operation == '-x':
if output_file:
file_name = output_file
else:
file_name = input_file.replace('.z78', '.txt')
decompress(input_file, file_name)
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(102400000)
print("%s send zipbytes %d" % (self.client_address[0], len(self.data)))
result = decompress(io.BytesIO(self.data))
# just send back the same data, but upper-cased
self.request.sendall(serialize(result))
def convert_txa(filename, out_dir):
data = ConstBitStream(filename=filename)
out_template = os.path.join(
out_dir,
os.path.splitext(os.path.basename(filename))[0])
try:
os.makedirs(out_template)
except:
pass
magic = data.read("bytes:4")
size = data.read("uintle:32")
indexed = data.read("uintle:32")
chunks = data.read("uintle:32")
dec_size = data.read("uintle:32")
unk2 = data.read("uintle:32")
unk3 = data.read("uintle:32")
unk4 = data.read("uintle:32")
print(size, chunks, indexed)
for i in range(chunks):
hdr_len = data.read("uintle:16")
index = data.read("uintle:16")
w = data.read("uintle:16")
h = data.read("uintle:16")
entry_off = data.read("uintle:32")
entry_len = data.read("uintle:32")
name = data.read("bytes:%d" % (hdr_len - 16)).strip("\0")
print(name, index, w, h, entry_off, entry_len)
temp_pos = data.bytepos
data.bytepos = entry_off
# If we are zero size, we're not compressed.
if entry_len == 0:
if indexed:
size = 1024 + (w * h)
chunk = bytearray(data.read(size * 8).bytes)
else:
print("???")
else:
chunk = data.read(entry_len * 8)
chunk = decompress(chunk)
data.bytepos = temp_pos
w = adjust_w(w)
if indexed:
chunk, masked = get_indexed(chunk, w, h, crop=False)
else:
chunk = get_rgba(chunk, w, h, crop=False)
chunk.save("%s/%s.png" % (out_template, name))
# dump_to_file(chunk, "temp%d.dat" % index)
################################################################################
def test_short():
table = {
'*': 'c',
'#': '00',
'$': '*y',
}
assert decompress('$3#', table) == 'cy300'
def test_decode_many_values(self):
assert (decompress([
(0, 12, "W"),
(0, 0, "B"),
(0, 12, "W"),
(0, 3, "B"),
(0, 24, "W"),
(0, 0, "B"),
]) == "WWWWWWWWWWWWBWWWWWWWWWWWWBBBWWWWWWWWWWWWWWWWWWWWWWWWB")
def test_long():
table = {
'#': 'hem',
'@': 'T#',
'$': 't#',
'&': '$ as',
'*': ' do ',
'%': ' to',
'^': ' someone ',
'~': 'for ',
'+': '~&',
}
assert decompress("@ as can*has%*+ can't. And^has% speak up + has no voices.", table) == "Them as can do has to " \
"do for them as can't. " \
"And someone has to " \
"speak up for them as " \
"has no voices."
def convert_msk(filename, out_file):
data = ConstBitStream(filename=filename)
magic = data.read("bytes:4")
size = data.read("uintle:32")
w = data.read("uintle:16")
h = data.read("uintle:16")
cmp_size = data.read("uintle:32")
print(w, h, cmp_size)
w = adjust_w(w)
chunk = data.read(cmp_size * 8)
chunk = decompress(chunk)
dump_to_file(chunk)
chunk = get_grayscale(chunk, w, h, crop=False)
chunk.save(out_file)
################################################################################
def test_multiple_character(self):
assert decompress(from_bytes(to_bytes(
compress("abcdefg")))) == "abcdefg"
def test_one_character(self):
assert decompress(from_bytes(to_bytes(compress("a")))) == "a"
def test_decompress_inverts_compress(s):
assert decompress(compress(s)) == s
def test_no_conversions():
assert decompress('Hello World!', {'*': 'o', '#': 'h'}) == 'Hello World!'
import os
from sendDataToDB import executeScriptsFromFile
from decompress import decompress
#Iterates over the directory 'outbox' and decompresses its files in 'data'
for filename in os.listdir('outbox'):
if not filename.endswith(".start.tar.bz2"):
decompress('outbox/' + filename)
os.rename('outbox/' + filename, 'sent/' + filename)
for filename in os.listdir('data'):
executeScriptsFromFile('data/' + filename)
os.remove('data/' + filename)
def __init__(self):
d.decompress('imagens/quarto')
self.ImagemSala = pygame.image.load('imagens/quarto.jpg')
self.Quarto = pygame.transform.scale(self.ImagemSala, (1366, 768))
self.rectQuarto = self.Quarto.get_rect()
def test_single_characters(self):
assert decompress([(0, 0, "x"), (0, 0, "y"), (0, 0, "z")]) == "xyz"
def main():
key, code = compress(sys.argv[1])
decompress(code, key)
def test_empty_table():
assert decompress('Hello World!', {}) == 'Hello World!'
def __init__(self):
d.decompress('imagens/sala')
self.ImagemSala = pygame.image.load('imagens/sala.jpg')
self.Sala = pygame.transform.scale(self.ImagemSala, (1366, 768))
self.rectSala = self.Sala.get_rect()
def test_empty_string():
assert decompress('', {'[', 'L'}) == ''
def process_chunk(data,
offset,
bothChunkSize=None,
forceColorTable0ToBlackTransparent=False):
if offset > data.len / 8:
return None, 0, 0
temp_pos = data.bytepos
data.bytepos = offset
chunk_type = data.read("uintle:16")
unk0 = data.read("uintle:16")
unk1a = data.read("uintle:16")
unk1b = data.read("uintle:16")
x = data.read("uintle:16")
y = data.read("uintle:16")
w = data.read("uintle:16")
h = data.read("uintle:16")
size = data.read("uintle:32")
unk3a = data.read("uintle:16")
unk3b = data.read("uintle:16")
unk4a = data.read("uintle:16")
unk4b = data.read("uintle:16")
unk5a = data.read("uintle:16")
unk5b = data.read("uintle:16")
if conf.debug:
print("Offset: ", offset)
print("Type, Unk0:", chunk_type, unk0)
print("Unk1: ", unk1a, unk1b)
print("X, Y: ", x, y)
print("W, H: ", w, h)
print("Size: ", size)
print("Unk3: ", unk3a, unk3b)
print("Unk4: ", unk4a, unk4b)
print("Unk5: ", unk5a, unk5b)
print()
# If size is zero, then we're uncompressed.
if size == 0:
# This is so f*****g hacky, but I don't care.
if chunk_type == 3 or chunk_type == 2:
if ((chunk_type == 0 or chunk_type == 2) and
(unk0 > 0 or not (unk5a == 0x00 and unk5b == 0x00))) or (
(chunk_type == 1 or chunk_type == 3) and
(unk0 > 0 and not (unk5a == 0x00 and unk5b == 0x00))):
while True:
data.read(12 * 8)
flag = data.read("uintle:32")
if flag == 0x00:
break
# 1024 for the palette, plus one for each pixel.
size = 1024 + (w * h)
chunk = bytearray(data.read(size * 8).bytes)
image, masked = get_indexed(chunk, w, h)
data.bytepos = temp_pos
return image, x, y, masked
data.bytepos = temp_pos
return None, 0, 0
# This is so f*****g hacky, but I don't care.
# if (chunk_type == 0 and unk0 > 0) or (chunk_type == 2):
if ((chunk_type == 0 or chunk_type == 2) and
(unk0 > 0 or not (unk5a == 0x00 and unk5b == 0x00))) or (
(chunk_type == 1 or chunk_type == 3) and
(unk0 > 0 and not (unk5a == 0x00 and unk5b == 0x00))):
data.bytepos += 16
while True:
cur_pos = data.bytepos
bytesToRead = size * 8
print("trying to use {} bytes from pos {}".format(
bytesToRead, cur_pos))
comp = data.read(bytesToRead)
try:
dec = decompress(comp)
if conf.debug:
print("Decoded Chunk Size: {}".format(len(dec)))
except:
# data.bytepos = cur_pos + 16
data.bytepos = cur_pos
while True:
data.read(12 * 8)
flag = data.read("uintle:32")
if flag == 0x00:
break
continue
break
# dump_bs_to_file(comp)
# dump_to_file(dec, "temp2.dat")
else:
comp = data.read(size * 8)
dec = decompress(comp)
adj_w = adjust_w(w)
# Indexed image.
if chunk_type == 3 or chunk_type == 2:
print("using indexed mode")
chunk, masked = get_indexed(dec,
adj_w,
h,
forceColorTable0ToBlackTransparent=
forceColorTable0ToBlackTransparent)
else:
print("using RGBA Mode")
chunk = get_rgba(dec, adj_w, h)
masked = True
# chunk_type = data.read("uintle:16")
# unk0 = data.read("uintle:16")
#
# unk1a = data.read("uintle:16")
# unk1b = data.read("uintle:16")
# x = data.read("uintle:16")
# y = data.read("uintle:16")
# w = data.read("uintle:16")
# h = data.read("uintle:16")
#
# size = data.read("uintle:32")
# unk3a = data.read("uintle:16")
# unk3b = data.read("uintle:16")
#
# unk4a = data.read("uintle:16")
# unk4b = data.read("uintle:16")
#
# unk5a = data.read("uintle:16")
# unk5b = data.read("uintle:16")
#
# if conf.debug:
# print("----extra chunk-----")
# print("Offset: N/A, assume comes after previous chunk")
# print("Type, Unk0:", chunk_type, unk0)
# print("Unk1: ", unk1a, unk1b)
# print("X, Y: ", x, y)
# print("W, H: ", w, h)
# print("Size: ", size)
# print("Unk3: ", unk3a, unk3b)
# print("Unk4: ", unk4a, unk4b)
# print("Unk5: ", unk5a, unk5b)
# print()
print("chunk ends at: ", data.bytepos)
# amountToRead = (bothChunkSize - size)
#
# print("processing extra chunk of size: ", amountToRead)
#
#
#
# comp = data.read(amountToRead * 8)
# dec = decompress(comp)
data.bytepos = temp_pos
return chunk, x, y, masked
def test_example():
table = {'$': 's', '%': 'y', '/': 't'}
assert decompress('P%Bi/e$', table) == 'PyBites'
def test_repeated_characters(self):
assert decompress(from_bytes(to_bytes(compress("aaaaa")))) == "aaaaa"
def test_simple_strings(self, s):
assert decompress(from_bytes(to_bytes(compress(s)))) == s
def test_decompress(code, result):
assert decompress(code) == result
def test_simple_case(self):
assert decompress([(0, 2, "A"), (0, 3, "B"),
(0, 4, "C")]) == "AABBBCCCC"
from argparse import ArgumentParser, FileType
from compress import compress
from decompress import decompress
if __name__ == "__main__":
parser = ArgumentParser(description = "Compress or decompress a file using a simple dictionary method.")
#could make better use of argparse to load the arguments as files directly
group = parser.add_mutually_exclusive_group(required = True)
group.add_argument("-d")
group.add_argument("-c")
parser.add_argument("output", nargs=1)
args = parser.parse_args()
outputfile = open(args.output[0], 'w')
if args.c:
inputfile = args.c
compress_this = open(inputfile.readlines())
outtext = compress(compress_this)
elif args.d:
inputfile = args.d
decompress_this = compress_this = open(inputfile.readlines())
outtext = decompress(decompress_this)
outputfile.write(outtext)
outputfile.close()
def process_chunk(data, offset):
if offset > data.len / 8:
return None, 0, 0
temp_pos = data.bytepos
data.bytepos = offset
chunk_type = data.read("uintle:16")
unk0 = data.read("uintle:16")
unk1a = data.read("uintle:16")
unk1b = data.read("uintle:16")
x = data.read("uintle:16")
y = data.read("uintle:16")
w = data.read("uintle:16")
h = data.read("uintle:16")
size = data.read("uintle:32")
unk3a = data.read("uintle:16")
unk3b = data.read("uintle:16")
unk4a = data.read("uintle:16")
unk4b = data.read("uintle:16")
unk5a = data.read("uintle:16")
unk5b = data.read("uintle:16")
print "Offset: ", offset
print "Type, Unk0:", chunk_type, unk0
print "Unk1: ", unk1a, unk1b
print "X, Y: ", x, y
print "W, H: ", w, h
print "Size: ", size
print "Unk3: ", unk3a, unk3b
print "Unk4: ", unk4a, unk4b
print "Unk5: ", unk5a, unk5b
print
# If size is zero, then we're uncompressed.
if size == 0:
# This is so f*****g hacky, but I don't care.
if chunk_type == 3 or chunk_type == 2:
if ((chunk_type == 0 or chunk_type == 2) and
(unk0 > 0 or not (unk5a == 0x00 and unk5b == 0x00))) or (
(chunk_type == 1 or chunk_type == 3) and
(unk0 > 0 and not (unk5a == 0x00 and unk5b == 0x00))):
while True:
data.read(12 * 8)
flag = data.read("uintle:32")
if flag == 0x00:
break
# 1024 for the palette, plus one for each pixel.
size = 1024 + (w * h)
chunk = bytearray(data.read(size * 8).bytes)
image, masked = get_indexed(chunk, w, h)
data.bytepos = temp_pos
return image, x, y, masked
data.bytepos = temp_pos
return None, 0, 0
# This is so f*****g hacky, but I don't care.
# if (chunk_type == 0 and unk0 > 0) or (chunk_type == 2):
if ((chunk_type == 0 or chunk_type == 2) and
(unk0 > 0 or not (unk5a == 0x00 and unk5b == 0x00))) or (
(chunk_type == 1 or chunk_type == 3) and
(unk0 > 0 and not (unk5a == 0x00 and unk5b == 0x00))):
data.bytepos += 16
while True:
cur_pos = data.bytepos
comp = data.read(size * 8)
try:
dec = decompress(comp)
except:
# data.bytepos = cur_pos + 16
data.bytepos = cur_pos
while True:
data.read(12 * 8)
flag = data.read("uintle:32")
if flag == 0x00:
break
continue
break
# dump_bs_to_file(comp)
# dump_to_file(dec, "temp2.dat")
else:
comp = data.read(size * 8)
dec = decompress(comp)
adj_w = adjust_w(w)
# Indexed image.
if chunk_type == 3 or chunk_type == 2:
chunk, masked = get_indexed(dec, adj_w, h)
else:
chunk = get_rgba(dec, adj_w, h)
masked = True
data.bytepos = temp_pos
return chunk, x, y, masked
def test_it_compresses_a_simple_example():
assert decompress(
compress("This is a simple test")) == "This is a simple test"
import sys, time
from pathlib import Path
from compress import compress
from decompress import decompress
f = open("data/document.txt", "r") # open the original document
content = ''.join(list(f)) # add every line to the "content" variable
f.close() #close the file after finishing reading it.
start_time = time.time() # capture time before compressing
_compressed = compress(content) # calling the compression algorithm
length_in_time = time.time(
) - start_time # capture time after compressing and meassure the differenec
_decompressed = decompress(_compressed) # calling the decompression algorithm
# print(_compressed)
# save the compressed version of the document
f = open("data/compressed.txt", "w+")
f.write(compress(content))
f.close()
# make sure source matches the decompressed (to avoid data loss)
if (_decompressed == content):
print("✅No data lost.")
else:
print(f"❌Some data was lost", _decompressed, content)
initial = Path('data/document.txt').stat().st_size
last = Path('data/compressed.txt').stat().st_size
percentage = 100 - round(last / initial * 100)
print(
comp_model = CompModel(args)
# use network interpolation
state_dict1 = torch.load(args.model_path, map_location='cpu')['comp_model']
state_dict2 = torch.load(args.model_path2, map_location='cpu')['comp_model']
state_dict_interp = OrderedDict()
for k in state_dict1:
if 'decoder' in k:
state_dict_interp[k] = args.alpha * state_dict1[k] + (1 - args.alpha) * state_dict2[k]
else:
state_dict_interp[k] = state_dict1[k]
comp_model.load_state_dict(state_dict_interp)
comp_model.eval()
return comp_model
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('model_path')
parser.add_argument('model_path2')
parser.add_argument('alpha', type=float)
parser.add_argument('binary_path')
parser.add_argument('--bottleneck', type=int, default=32)
parser.add_argument('--main_channel', type=int, default=192)
parser.add_argument('--gmm_K', type=int, default=3)
args = parser.parse_args()
comp_model = load_model(args)
decompress(comp_model, args)