def test_with_large_strings(self):
test_sizes = [1000, 10000, 100000]
for size in test_sizes:
contents = random_string(size).encode('ascii')
decompressed = heatshrink.decompress(heatshrink.compress(contents))
self.assertEqual(decompressed, contents)
def compress(self):
if self.flag("COMPRESS"):
print("WARN: Image already compressed!")
return
uncompressed_length = self.file_size - self.get_header_length()
self.fd.seek(self.get_header_length(), os.SEEK_SET)
data = self.fd.read(uncompressed_length)
data = heatshrink2.compress(data, window_sz2=11, lookahead_sz2=4)
self.fd.seek(self.get_header_length(), os.SEEK_SET)
self.fd.truncate(self.get_header_length())
self.fd.write(data)
self.flag("COMPRESS", True)
self.read_file_size()
def setUp(self):
self.compressed = heatshrink.compress(b'abcde')
def test_round_trip(self):
compressed = heatshrink.compress(b'a string')
self.assertEqual(heatshrink.decompress(compressed), b'a string')
def test_with_a_paragraph(self):
compressed = heatshrink.compress(TEXT)
self.assertEqual(heatshrink.decompress(compressed), TEXT)
def test_different_params_yield_different_output(self):
string = b'A string with stuff in it'
self.assertNotEqual(heatshrink.compress(string, window_sz2=8),
heatshrink.compress(string, window_sz2=11))
self.assertNotEqual(heatshrink.compress(string, lookahead_sz2=4),
heatshrink.compress(string, lookahead_sz2=8))
def test_compress_with_lookahead_sz2(self):
compressed = heatshrink.compress(b'abcde', lookahead_sz2=3)
self.assertEqual(compressed, b'\xb0\xd8\xacvK(')
def test_compress_with_window_sz2(self):
compressed = heatshrink.compress(b'abcde', window_sz2=8)
# FIXME: Prove that this setting changes output
self.assertEqual(compressed, b'\xb0\xd8\xacvK(')
def add_file(config, root, filename, actions):
filepath = os.path.join(root, filename)
with open(filepath, 'rb') as f:
initial_data = f.read()
processed_data = initial_data
for action in actions:
if action in ['gzip', 'heatshrink']:
pass
elif action in config['tools']:
tool = config['tools'][action]
command = tool['command']
p = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
processed_data = p.communicate(input=processed_data)[0]
else:
print(f'Unknown action: {action}', file=sys.stderr)
sys.exit(1)
flags = 0
if 'gzip' in actions:
flags |= FLAG_GZIP
tool = config['tools']['gzip']
level = min(max(tool.get('level', 9), 0), 9)
processed_data = gzip.compress(processed_data, level)
if 'heatshrink' in actions:
compression = COMPRESS_HEATSHRINK
tool = config['tools']['heatshrink']
level = min(max(tool.get('level', 9), 0), 9) // 2
window_sizes, lookahead_sizes = [5, 6, 8, 11, 13], [3, 3, 4, 4, 4]
window_sz2, lookahead_sz2 = window_sizes[level], lookahead_sizes[level]
header = bytes([window_sz2 << 4 | lookahead_sz2])
compressed_data = header + heatshrink2.compress(
processed_data, window_sz2=window_sz2, lookahead_sz2=lookahead_sz2)
else:
compression = COMPRESS_NONE
compressed_data = processed_data
if len(compressed_data) >= len(processed_data):
compression = COMPRESS_NONE
compressed_data = processed_data
initial_len, processed_len, compressed_len = len(initial_data), len(
processed_data), len(compressed_data)
if initial_len < 1024:
initial = f'{initial_len} B'
compressed = f'{compressed_len} B'
elif initial_len < 1024 * 1024:
initial = f'{initial_len / 1024:.1f} KiB'
compressed = f'{compressed_len / 1024:.1f} KiB'
elif initial_len < 1024 * 1024 * 1024:
initial = f'{initial_len / 1024 / 1024:.1f} MiB'
compressed = f'{compressed_len / 1024 / 1024:.1f} MiB'
percent = 100.0
if initial_len > 0:
percent = compressed_len / initial_len * 100
filename = filename.replace('\\', '/')
print(f'{filename}: {initial} -> {compressed} ({percent:.1f}%)')
filename = filename.encode('utf8')
filename = filename.ljust((len(filename) + 4) // 4 * 4, b'\0')
compressed_data = compressed_data.ljust((compressed_len + 3) // 4 * 4,
b'\0')
header = header_struct.pack(ESPFS_MAGIC, flags, compression, len(filename),
compressed_len, processed_len)
return header + filename + compressed_data
def make_file_object(hash, path, data, attributes):
global config
index = attributes['index']
actions = attributes['actions']
flags = 0
compression = ESPFS_COMPRESSION_NONE
initial_data = data
initial_len = len(data)
if 'cache' in actions:
flags |= ESPFS_FLAG_CACHE
for action in actions:
if action in config['preprocessors']:
command = config['preprocessors'][action]['command']
process = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
data = process.communicate(input=data)[0]
file_data = data
file_len = len(data)
if file_len >= initial_len:
data = initial_data
file_len = initial_len
if 'gzip' in actions:
flags |= ESPFS_FLAG_GZIP
level = config['compressors']['gzip']['level']
level = min(max(level, 0), 9)
data = gzip.compress(data, level)
elif 'heatshrink' in actions:
compression = ESPFS_COMPRESSION_HEATSHRINK
window_sz2 = config['compressors']['heatshrink']['window_sz2']
lookahead_sz2 = config['compressors']['heatshrink']['lookahead_sz2']
data = espfs_heatshrink_header_t.pack(
window_sz2, lookahead_sz2, 0) + heatshrink2.compress(
data, window_sz2=window_sz2, lookahead_sz2=lookahead_sz2)
data_len = len(data)
if data_len >= file_len:
flags &= ~ESPFS_FLAG_GZIP
compression = ESPFS_COMPRESSION_NONE
data = file_data
data_len = file_len
if initial_len < 1024:
initial_len_str = '%d B' % (initial_len)
data_len_str = '%d B' % (data_len)
elif initial_len < 1024 * 1024:
initial_len_str = '%.1f KiB' % (initial_len / 1024)
data_len_str = '%.1f KiB' % (data_len / 1024)
else:
initial_len_str = '%.1f MiB' % (initial_len / 1024 / 1024)
data_len_str = '%.1f MiB' % (data_len / 1024 / 1024)
percent = 100.0
if initial_len > 0:
percent = data_len / initial_len * 100.0
stats = '%-9s -> %-9s (%.1f%%)' % (initial_len_str, data_len_str, percent)
print('%08x %-34s file %s' % (hash, path, stats))
path = path.encode('utf8') + b'\0'
path = path.ljust((len(path) + 3) // 4 * 4, b'\0')
data = data.ljust((data_len + 3) // 4 * 4, b'\0')
header = espfs_object_header_t.pack(
ESPFS_TYPE_FILE, espfs_object_header_t.size + espfs_file_header_t.size,
index, len(path), 0) + espfs_file_header_t.pack(
data_len, file_len, flags, compression, 0)
return header + path + data