def brieflz_compdecsingle(data):
import brieflz
c = brieflz.compress(data)
compressed = c.do()
d = brieflz.decompress(compressed, len(compressed))
decompressed , consumed = d.do()
return compressed, decompressed
def write_languages(
data: LanguageData,
f: TextIO,
strings_obj_path: Optional[str] = None,
compress_font: bool = False,
) -> None:
defs = data.defs
build_version = data.build_version
combined_sym_list = data.sym_list
sym_lists_by_font = data.sym_lists_by_font
font_map = data.font_map
symbol_conversion_table = build_symbol_conversion_map(combined_sym_list)
language_codes: List[str] = [lang["languageCode"] for lang in data.langs]
logging.info(f"Generating block for {language_codes}")
lang_names = [
lang.get("languageLocalName", lang["languageCode"])
for lang in data.langs
]
f.write('#include "Translation_multi.h"')
f.write(f"\n// ---- {lang_names} ----\n\n")
max_decompressed_font_size = 0
if not compress_font:
font_table_text = ""
font_section_info_text = (
"const FontSectionDataInfo FontSectionDataInfos[] = {\n")
for font, current_sym_list in sym_lists_by_font.items():
font_table_text += f"const uint8_t font_table_data_{font}[] = {{\n"
font_table_text += "// 12x16:\n"
font_table_text += make_font_table_named_cpp(
None,
current_sym_list,
font_map.font12,
symbol_conversion_table,
)
if font != font_tables.NAME_CJK:
font_table_text += "// 6x8:\n"
font_table_text += make_font_table_named_cpp(
None,
current_sym_list,
font_map.font06, # type: ignore[arg-type]
symbol_conversion_table,
)
font_table_text += f"}}; // font_table_data_{font}\n"
current_sym_start = combined_sym_list.index(
current_sym_list[0]) + 2
font_section_info_text += (
" {\n"
f" .symbol_start = {current_sym_start},\n"
f" .symbol_count = {len(current_sym_list)},\n"
f" .data_size = sizeof(font_table_data_{font}),\n"
" .data_is_compressed = false,\n"
f" .data_ptr = font_table_data_{font},\n"
" },\n")
f.write(font_table_text)
font_section_info_text += (
"};\n"
"const uint8_t FontSectionDataCount = sizeof(FontSectionDataInfos) / sizeof(FontSectionDataInfos[0]);\n\n"
)
f.write(font_section_info_text)
f.write(
"FontSection DynamicFontSections[4] = {};\n"
"const FontSection *const FontSections = DynamicFontSections;\n"
"const uint8_t FontSectionsCount = sizeof(DynamicFontSections) / sizeof(DynamicFontSections[0]);\n"
)
else:
font_section_info_text = (
"const FontSectionDataInfo FontSectionDataInfos[] = {\n")
for font, current_sym_list in sym_lists_by_font.items():
current_sym_start = combined_sym_list.index(
current_sym_list[0]) + 2
font_uncompressed = bytearray()
for sym in current_sym_list:
font_uncompressed.extend(font_map.font12[sym])
if font != font_tables.NAME_CJK:
for sym in current_sym_list:
font_uncompressed.extend(
font_map.font06[sym]) # type: ignore[arg-type]
font_compressed = brieflz.compress(bytes(font_uncompressed))
logging.info(
f"Font table for {font} compressed from {len(font_uncompressed)} to {len(font_compressed)} bytes (ratio {len(font_compressed) / len(font_uncompressed):.3})"
)
max_decompressed_font_size += len(font_uncompressed)
write_bytes_as_c_array(f, f"font_data_brieflz_{font}",
font_compressed)
font_section_info_text += (
" {\n"
f" .symbol_start = {current_sym_start},\n"
f" .symbol_count = {len(current_sym_list)},\n"
f" .data_size = sizeof(font_data_brieflz_{font}),\n"
" .data_is_compressed = true,\n"
f" .data_ptr = font_data_brieflz_{font},\n"
" },\n")
font_section_info_text += (
"};\n"
"const uint8_t FontSectionDataCount = sizeof(FontSectionDataInfos) / sizeof(FontSectionDataInfos[0]);\n\n"
)
f.write(font_section_info_text)
f.write(
"FontSection DynamicFontSections[4] = {};\n"
"const FontSection *const FontSections = DynamicFontSections;\n"
"const uint8_t FontSectionsCount = sizeof(DynamicFontSections) / sizeof(DynamicFontSections[0]);\n"
)
f.write(f"\n// ---- {lang_names} ----\n\n")
translation_common_text = get_translation_common_text(
defs, symbol_conversion_table, build_version)
f.write(translation_common_text)
f.write(
f"const bool HasFahrenheit = {('true' if any([lang.get('tempUnitFahrenheit', True) for lang in data.langs]) else 'false')};\n\n"
)
max_decompressed_translation_size = 0
if not strings_obj_path:
for lang in data.langs:
lang_code = lang["languageCode"]
translation_strings_and_indices_text = (
get_translation_strings_and_indices_text(
lang,
defs,
symbol_conversion_table,
suffix=f"_{lang_code}"))
f.write(translation_strings_and_indices_text)
f.write("const LanguageMeta LanguageMetas[] = {\n")
for lang in data.langs:
lang_code = lang["languageCode"]
lang_id = get_language_unqiue_id(lang_code)
f.write(
" {\n"
f" .uniqueID = {lang_id},\n"
f" .translation_data = reinterpret_cast<const uint8_t *>(&translation_{lang_code}),\n"
f" .translation_size = sizeof(translation_{lang_code}),\n"
f" .translation_is_compressed = false,\n"
" },\n")
f.write("};\n")
else:
for lang in data.langs:
lang_code = lang["languageCode"]
sym_name = objcopy.cpp_var_to_section_name(
f"translation_{lang_code}")
strings_bin = objcopy.get_binary_from_obj(strings_obj_path,
sym_name)
if len(strings_bin) == 0:
raise ValueError(f"Output for {sym_name} is empty")
max_decompressed_translation_size = max(
max_decompressed_translation_size, len(strings_bin))
compressed = brieflz.compress(strings_bin)
logging.info(
f"Strings for {lang_code} compressed from {len(strings_bin)} to {len(compressed)} bytes (ratio {len(compressed) / len(strings_bin):.3})"
)
write_bytes_as_c_array(f, f"translation_data_brieflz_{lang_code}",
compressed)
f.write("const LanguageMeta LanguageMetas[] = {\n")
for lang in data.langs:
lang_code = lang["languageCode"]
lang_id = get_language_unqiue_id(lang_code)
f.write(
" {\n"
f" .uniqueID = {lang_id},\n"
f" .translation_data = translation_data_brieflz_{lang_code},\n"
f" .translation_size = sizeof(translation_data_brieflz_{lang_code}),\n"
f" .translation_is_compressed = true,\n"
" },\n")
f.write("};\n")
f.write(
"const uint8_t LanguageCount = sizeof(LanguageMetas) / sizeof(LanguageMetas[0]);\n\n"
f"alignas(TranslationData) uint8_t translation_data_out_buffer[{max_decompressed_translation_size + max_decompressed_font_size}];\n"
"const uint16_t translation_data_out_buffer_size = sizeof(translation_data_out_buffer);\n\n"
)
sanity_checks_text = get_translation_sanity_checks_text(defs)
f.write(sanity_checks_text)
def write_language(
data: LanguageData,
f: TextIO,
strings_bin: Optional[bytes] = None,
compress_font: bool = False,
) -> None:
if len(data.langs) > 1:
raise ValueError("More than 1 languages are provided")
lang = data.langs[0]
defs = data.defs
build_version = data.build_version
sym_list = data.sym_list
font_map = data.font_map
symbol_conversion_table = build_symbol_conversion_map(sym_list)
language_code: str = lang["languageCode"]
logging.info(f"Generating block for {language_code}")
try:
lang_name = lang["languageLocalName"]
except KeyError:
lang_name = language_code
if strings_bin or compress_font:
f.write('#include "brieflz.h"\n')
f.write(f"\n// ---- {lang_name} ----\n\n")
if not compress_font:
font_table_text = make_font_table_cpp(sym_list, font_map,
symbol_conversion_table)
f.write(font_table_text)
f.write(
"const FontSection FontSectionsData[] = {\n"
" {\n"
" .symbol_start = 2,\n"
f" .symbol_end = {len(sym_list) + 2},\n"
" .font12_start_ptr = USER_FONT_12,\n"
" .font06_start_ptr = USER_FONT_6x8,\n"
" },\n"
"};\n"
"const FontSection *const FontSections = FontSectionsData;\n"
"const uint8_t FontSectionsCount = sizeof(FontSectionsData) / sizeof(FontSectionsData[0]);\n"
)
else:
font12_uncompressed = bytearray()
for sym in sym_list:
font12_uncompressed.extend(font_map.font12[sym])
font12_compressed = brieflz.compress(bytes(font12_uncompressed))
logging.info(
f"Font table 12x16 compressed from {len(font12_uncompressed)} to {len(font12_compressed)} bytes (ratio {len(font12_compressed) / len(font12_uncompressed):.3})"
)
write_bytes_as_c_array(f, "font_12x16_brieflz", font12_compressed)
font_table_text = make_font_table_06_cpp(sym_list, font_map,
symbol_conversion_table)
f.write(font_table_text)
f.write(
f"static uint8_t font_out_buffer[{len(font12_uncompressed)}];\n"
"const FontSection FontSectionsData[] = {\n"
" {\n"
" .symbol_start = 2,\n"
f" .symbol_end = {len(sym_list) + 2},\n"
" .font12_start_ptr = font_out_buffer,\n"
" .font06_start_ptr = USER_FONT_6x8,\n"
" },\n"
"};\n"
"const FontSection *const FontSections = FontSectionsData;\n"
"const uint8_t FontSectionsCount = sizeof(FontSectionsData) / sizeof(FontSectionsData[0]);\n"
)
f.write(f"\n// ---- {lang_name} ----\n\n")
translation_common_text = get_translation_common_text(
defs, symbol_conversion_table, build_version)
f.write(translation_common_text)
f.write(
f"const bool HasFahrenheit = {('true' if lang.get('tempUnitFahrenheit', True) else 'false')};\n\n"
)
if not strings_bin:
translation_strings_and_indices_text = get_translation_strings_and_indices_text(
lang, defs, symbol_conversion_table)
f.write(translation_strings_and_indices_text)
f.write("const TranslationIndexTable *Tr = &translation.indices;\n"
"const char *TranslationStrings = translation.strings;\n\n")
else:
compressed = brieflz.compress(strings_bin)
logging.info(
f"Strings compressed from {len(strings_bin)} to {len(compressed)} bytes (ratio {len(compressed) / len(strings_bin):.3})"
)
write_bytes_as_c_array(f, "translation_data_brieflz", compressed)
f.write(
f"static uint8_t translation_data_out_buffer[{len(strings_bin)}] __attribute__((__aligned__(2)));\n\n"
"const TranslationIndexTable *Tr = reinterpret_cast<const TranslationIndexTable *>(translation_data_out_buffer);\n"
"const char *TranslationStrings = reinterpret_cast<const char *>(translation_data_out_buffer) + sizeof(TranslationIndexTable);\n\n"
)
if not strings_bin and not compress_font:
f.write("void prepareTranslations() {}\n\n")
else:
f.write("void prepareTranslations() {\n")
if compress_font:
f.write(
" blz_depack_srcsize(font_12x16_brieflz, font_out_buffer, sizeof(font_12x16_brieflz));\n"
)
if strings_bin:
f.write(
" blz_depack_srcsize(translation_data_brieflz, translation_data_out_buffer, sizeof(translation_data_brieflz));\n"
)
f.write("}\n\n")
sanity_checks_text = get_translation_sanity_checks_text(defs)
f.write(sanity_checks_text)
def test_roundtrip(self):
packed = brieflz.compress(TEST_DATA)
depacked = brieflz.depack_srcsize(packed, len(TEST_DATA))
self.assertEqual(depacked, TEST_DATA)