def extract(path, out=None):
if out is None:
out = path.replace('.', '_')
if out == path:
out += '_arc'
with open(path, 'rb') as f:
_magic, _version_maybe, filecount, _compression_maybe = \
unpack_f('IIII', f)
files = []
for _ in range(filecount):
str_offset, file_offset, unpacked_size, packed_size = \
unpack_f('IIII', f)
files.append([str_offset, file_offset, unpacked_size, packed_size])
for entry in files:
f.seek(entry[0])
entry[0] = grab_string(f)
for name, file_offset, unpacked_size, packed_size in tqdm(files):
tqdm.write(name)
file = os.path.join(out, name)
folder = os.path.dirname(file)
mkdir_p(folder)
with open(file, 'wb') as f_out:
f.seek(file_offset)
data = f.read(packed_size)
if unpacked_size != packed_size:
data = decompress(data)
if len(data) != unpacked_size:
tqdm.write('{} : {:08X}'.format(name, file_offset))
tqdm.write('MISMATCH: {0} ({0:02x}) {1} ({1:02x})'.format(
len(data), unpacked_size))
f_out.write(data)
def main():
print(sys.argv)
start_time = time.time()
if len(sys.argv) != 5:
printUsage()
if sys.argv[1] == "compress":
compress(filename=sys.argv[2],
path_to_dir=sys.argv[3],
output_dir=sys.argv[4])
end_information(sys.argv[3] + sys.argv[2], sys.argv[4] + sys.argv[2])
elif sys.argv[1] == "decompress":
decompress(filename=sys.argv[2],
path_to_dir=sys.argv[3],
output_dir=sys.argv[4])
else:
printUsage()
print(f"{sys.argv[1]} took {time.time() - start_time} seconds.")
def check_valid(rom, pointer, strict=False):
offset = get_rom_addr(pointer)
if offset > len(rom) - 5:
return False
firstbyte = rom[offset]
size = to_int(rom[offset + 1 : offset + 4])
if firstbyte != FIRSTBYTE_LZ77 or size < 32:
return False
elif size < 32 * 1024: # min 8x8 pixel, max 32kb
if strict:
try:
lz77.decompress(rom, offset)
return True
except:
return False
else:
return True
else:
return False
def tests():
#files = ['pan-tadeusz-czyli-ostatni-zajazd-na-litwie.txt',
# 'test1.bin',
# 'test2.bin',
# 'test3.bin']
files = ['my_test5.txt']
for filename in files:
start_time = time.time()
print(f"Start compressing {filename}")
compress(filename, path_to_dir="files/", output_dir="compressed/")
compress_time = time.time()
end_information("files/" + filename, "compressed/" + filename)
print(
f"Compression of {filename} took {round(compress_time - start_time,2)} seconds"
)
decompress(filename,
path_to_dir="compressed/",
output_dir="decompressed/")
decompress_time = time.time()
print(
f"Decompression of {filename} took {round(decompress_time - compress_time, 2)} seconds"
)
def getfile(self, path):
for node in self.files:
if node == path or (type(path) == str
and node.name.endswith(path)):
if node == path: path = node.name
self.file.seek(node.data_offset)
file = StringIO(self.file.read(node.size))
if path.startswith("LZ77"):
try:
decompressed_file = lz77.decompress(file)
file.close()
return decompressed_file
except ValueError, IndexError:
print "LZ77 decompression of '%s' failed" % path
print 'Dumping compressed file to %s' % path
f2 = open(path, "wb")
f2.write(file.read())
f2.close()
file.close()
return None
elif path.startswith("Huf8"):
try:
decompressed_file = StringIO()
huf8.decompress(file, decompressed_file)
file.close()
decompressed_file.seek(0)
return decompressed_file
except Exception:
print "Huf8 decompression of '%s' failed" % path
print "Dumping compressed file to %s" % path
f2 = open(path, "wb")
f2.write(file.read())
f2.close()
file.close()
return decompressed_file
elif path.startswith("LZH8"):
try:
decompressed_file = StringIO()
decompressed_file.write(lzh8.decompress(file))
decompressed_file.seek(0)
file.close()
return decompressed_file
except Exception:
print "LZH8 decompression of '%s' failed" % path
print "Dumping compressed file to %s" % path
f2 = open(path, "wb")
f2.write(file.read())
f2.close()
file.close()
else:
return file
def getfile(self, path):
for node in self.files:
if node == path or (type(path) == str and node.name.endswith(path)):
if node == path: path = node.name
self.file.seek(node.data_offset)
file = StringIO(self.file.read(node.size))
if path.startswith("LZ77"):
try:
decompressed_file = lz77.decompress(file)
file.close()
return decompressed_file
except ValueError, IndexError:
print "LZ77 decompression of '%s' failed" % path
print 'Dumping compressed file to %s' % path
f2 = open(path, "wb")
f2.write(file.read())
f2.close()
file.close()
return None
elif path.startswith("Huf8"):
try:
decompressed_file = StringIO()
huf8.decompress(file, decompressed_file)
file.close()
decompressed_file.seek(0)
return decompressed_file
except Exception:
print "Huf8 decompression of '%s' failed" % path
print "Dumping compressed file to %s" % path
f2 = open(path, "wb")
f2.write(file.read())
f2.close()
file.close()
return decompressed_file
elif path.startswith("LZH8"):
try:
decompressed_file = StringIO()
decompressed_file.write(lzh8.decompress(file))
decompressed_file.seek(0)
file.close()
return decompressed_file
except Exception:
print "LZH8 decompression of '%s' failed" % path
print "Dumping compressed file to %s" % path
f2 = open(path, "wb")
f2.write(file.read())
f2.close()
file.close()
else:
return file
import lz77
tests = [{
"message":
"кибернетики",
"buffer_size":
4,
"dict_size":
12,
"result": [(0, 0, 'к'), (0, 0, 'и'), (0, 0, 'б'), (0, 0, 'е'), (0, 0, 'р'),
(0, 0, 'н'), (3, 1, 'т'), (7, 1, 'к'), (2, 1, '')]
}, {
"message": "аааааааааааа",
"buffer_size": 4,
"dict_size": 12,
"result": [(0, 0, 'а'), (1, 1, 'а'), (3, 3, 'а'), (4, 4, 'а')]
}]
for test in tests:
result = lz77.compress(test['message'], test['buffer_size'],
test['dict_size'])
print(result)
assert result == test['result']
message = lz77.decompress(test['result'])
print(message)
assert message == test['message']
print('===========================')
def decode_tilemap_at(rom, codec, image_ptr, palettes_ptr, tilemap_ptr, width=16):
image_offset = get_rom_addr(image_ptr)
palettes_offset = get_rom_addr(palettes_ptr)
tilemap_offset = get_rom_addr(tilemap_ptr)
bpp = codec.getBitsPerPixel()
if check_valid(rom, image_offset, strict=True):
print("Loading compressed image at", hex(image_ptr))
image_data = lz77.decompress(rom, image_offset)
else:
print("Loading uncompressed image at", hex(image_ptr))
image_len = 1024 * codec.getTileSize() # Max amount of tiles
image_data = rom[image_offset : image_offset + image_len]
tile_size = codec.getTileSize()
if len(image_data) % tile_size:
# Fill remaining tile with padding bytes
image_data += b"\x00" * (tile_size - len(image_data) % tile_size)
pal_size = gba.get_palette_size(bpp)
if check_valid(rom, palettes_offset, strict=True):
print("Loading compressed palettes at", hex(palettes_ptr))
palettes_data = lz77.decompress(rom, palettes_offset)
if len(palettes_data) < pal_size:
return "PALETTE_TOO_SMALL"
else:
print("Loading uncompressed palettes at", hex(palettes_ptr))
palettes_len = 16 * pal_size # Max amount of palettes
palettes_data = rom[palettes_offset : palettes_offset + palettes_len]
if check_valid(rom, tilemap_offset, strict=True):
print("Loading compressed tilemap at", hex(tilemap_ptr))
tilemap = lz77.decompress(rom, tilemap_offset)
else:
print("Loading uncompressed tilemap at", hex(tilemap_ptr))
tilemap_len = 1024 # Max tilemap size (maybe)
tilemap = rom[tilemap_offset : tilemap_offset + tilemap_len]
try:
tiles = list(gba.iter_decode_tiles(codec, image_data))
except:
return "IMAGE_DECODE_FAILED"
try:
palettes = list(gba.iter_decode_palettes(palettes_data, bpp=bpp, alpha=True))
except:
return "PALETTE_DECODE_FAILED"
try:
tilemap_tiles = gba.decode_tilemap(tilemap, tiles, palettes)
img = gba.combine_tiles(tilemap_tiles, width)
except:
return "TILEMAP_DECODE_FAILED"
return img
def decode_image_at(rom, codec, image_ptr, palette_ptr, image_len=None, width=16):
image_offset = get_rom_addr(image_ptr)
palette_offset = get_rom_addr(palette_ptr)
bpp = codec.getBitsPerPixel()
if check_valid(rom, image_offset, strict=True):
print("Loading compressed image at", hex(image_ptr))
image_data = lz77.decompress(rom, image_offset)
elif image_len is not None:
print("Loading uncompressed image at", hex(image_ptr))
image_data = rom[image_offset : image_offset + image_len]
else:
return "MISSING_IMAGE_LENGTH"
tile_size = codec.getTileSize()
if len(image_data) % tile_size:
# Fill remaining tile with padding bytes
image_data += b"\x00" * (tile_size - len(image_data) % tile_size)
pal_size = gba.get_palette_size(bpp)
if check_valid(rom, palette_offset, strict=True):
print("Loading compressed palette at", hex(palette_ptr))
decomp_data = lz77.decompress(rom, palette_offset)
if len(decomp_data) < pal_size:
return "PALETTE_TOO_SMALL"
palette_data = decomp_data[:pal_size]
else:
print("Loading uncompressed palette at", hex(palette_ptr))
palette_data = rom[palette_offset : palette_offset + pal_size]
try:
palette = gba.decode_palette(palette_data, bpp=bpp)
except:
return "PALETTE_DECODE_FAILED"
try:
image = gba.decode_image(image_data, codec, palette, width)
except:
return "IMAGE_DECODE_FAILED"
return image