def _write_from_structure(self,
filename,
write_in_bytes=True,
compress=True,
log_writing=True,
log_reconstructing=False):
if hasattr(self, '_object_manager'):
self._object_manager.reconstruct(
log_reconstructing=log_reconstructing)
lgr = SimpleLogger(log_writing)
lgr.print("\nFile writing from structure started...")
byte_header_list = []
byte_data_list = []
for key in self._parsed_header:
lgr.print("\twriting " + key + "...", replace_line=True)
for retriever in self._parsed_header[key].retrievers:
byte_header_list.append(parser.retriever_to_bytes(retriever))
lgr.print("\twriting " + key + " finished successfully.",
replace_line=True)
lgr.print()
for key in self._parsed_data:
lgr.print("\twriting " + key + "...", replace_line=True)
for retriever in self._parsed_data[key].retrievers:
try:
byte_data_list.append(parser.retriever_to_bytes(retriever))
except AttributeError as e:
print("AttributeError occurred while writing '" + key +
"' > '" + retriever.name + "'")
print("\n\n\nAn error occurred. Writing failed.")
raise e
lgr.print("\twriting " + key + " finished successfully.",
replace_line=True)
lgr.print()
file = open(filename, "wb" if write_in_bytes else "w")
byte_header = b''.join(byte_header_list)
byte_data = b''.join(byte_data_list)
file.write(byte_header if write_in_bytes else create_textual_hex(
byte_header.hex()))
if compress:
lgr.print("\tCompressing...", replace_line=True)
# https://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760
deflate_obj = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS)
compressed = deflate_obj.compress(
b''.join(byte_data_list)) + deflate_obj.flush()
file.write(compressed if write_in_bytes else create_textual_hex(
compressed.hex()))
lgr.print("\tCompressing finished successfully.",
replace_line=True)
lgr.print()
else:
file.write(byte_data if write_in_bytes else create_textual_hex(
byte_data.hex()))
file.close()
lgr.print("File writing finished successfully.")
def get_byte_structure_as_string(self, skip_retrievers=None):
if skip_retrievers is None:
skip_retrievers = []
byte_structure = "\n" + self.get_header_string()
for retriever in self.retrievers:
if retriever.name in skip_retrievers:
continue
byte_structure += "\n"
if type(retriever.data) is list and len(retriever.data) > 0:
if isinstance(retriever.data[0], AoE2Piece):
for struct in retriever.data:
byte_structure += struct.get_byte_structure_as_string()
continue
else:
if isinstance(retriever.data, AoE2Piece):
byte_structure += retriever.data.get_byte_structure_as_string(
)
continue
retriever_data_bytes = parser.retriever_to_bytes(retriever)
if retriever_data_bytes is None:
return byte_structure
else:
retriever_data_bytes = retriever_data_bytes.hex()
retriever_short_string = retriever.get_short_str()
retriever_data_hex = helper.create_textual_hex(
retriever_data_bytes, space_distance=2, enter_distance=24)
if "\n" in retriever_data_hex:
split_hex = retriever_data_hex.split("\n")
if "\r\n" in retriever_short_string:
split_data_string = retriever_short_string.split("\r\n")
split_hex_length = len(split_hex)
split_data_string_length = len(split_data_string)
lines = max(split_hex_length, split_data_string_length)
combined_strings = []
for i in range(0, lines):
combined_strings.append(
helper.add_suffix_chars(
split_hex[i] if i < split_hex_length else "",
" ", 28) +
(split_data_string[i]
if i < split_data_string_length else ""))
byte_structure += "\n".join(combined_strings)
else:
split_hex[0] = helper.add_suffix_chars(
split_hex[0], " ", 28) + retriever_short_string
byte_structure += "\n".join(split_hex)
else:
byte_structure += helper.add_suffix_chars(
retriever_data_hex, " ", 28) + retriever_short_string
return byte_structure + "\n\n"
def get_byte_structure_as_string(self, pieces, skip_retrievers=None):
if skip_retrievers is None:
skip_retrievers = []
byte_structure = "\n" + self.get_header_string()
for retriever in self.retrievers:
if retriever.name in skip_retrievers:
continue
byte_structure += "\n"
listed_retriever_data = parser.listify(retriever.data)
struct_header_set = False
for struct in listed_retriever_data:
if isinstance(struct, AoE2Piece):
if not struct_header_set:
byte_structure += f"\n{'#' * 27} {retriever.name} ({retriever.datatype.to_simple_string()})"
struct_header_set = True
byte_structure += struct.get_byte_structure_as_string(pieces)
# Struct Header was set. Retriever was struct, data retrieved using recursion. Next retriever.
if struct_header_set:
byte_structure += f"{'#' * 27} End of: {retriever.name} ({retriever.datatype.to_simple_string()})\n"
continue
retriever_data_bytes = parser.retriever_to_bytes(retriever, pieces)
if retriever_data_bytes is None:
return byte_structure
else:
retriever_data_bytes = retriever_data_bytes.hex()
retriever_short_string: str = retriever.get_short_str()
retriever_hex = helper.create_textual_hex(
retriever_data_bytes, space_distance=2, enter_distance=24
)
split_hex = retriever_hex.split("\n")
split_hex_length = len(split_hex)
split_data_string = retriever_short_string.replace('\x00', ' ').splitlines()
data_lines = []
for x in split_data_string:
if len(x) > 120:
data_lines += [f'\t{x}' for x in helper.insert_char(x, '\r\n', 120).splitlines()]
else:
data_lines.append(x)
split_data_length = len(data_lines)
lines = max(split_hex_length, split_data_length)
combined_strings = []
for i in range(0, lines):
hex_part = split_hex[i] if i < split_hex_length else ""
data_part = data_lines[i] if i < split_data_length else ""
combined_strings.append(helper.add_suffix_chars(hex_part, " ", 28) + data_part)
byte_structure += "\n".join(combined_strings)
return byte_structure + "\n"
def _debug_write_from_source(self,
filename,
datatype,
write_in_bytes=True):
"""This function is used as a test debugging writing. It writes parts of the read file to the filesystem."""
print("File writing from source started with attributes " + datatype +
"...")
file = open(filename, "wb" if write_in_bytes else "w")
for t in datatype:
if t == "f":
file.write(self._file if write_in_bytes else
create_textual_hex(self._file.hex()))
elif t == "h":
file.write(self._file_header if write_in_bytes else
create_textual_hex(self._file_header.hex()))
elif t == "d":
file.write(self._file_data if write_in_bytes else
create_textual_hex(self._file_data.hex()))
file.close()
print("File writing finished successfully.")
def repeat_generator(generator, run_times, intended_stop_iteration=False, return_bytes=True):
elements = b'' if return_bytes else []
try:
for i in range(0, run_times):
if return_bytes:
elements += next(generator)
else:
# Generator returns list. So lists are merged
elements += next(generator)
except StopIteration as e:
if not intended_stop_iteration:
print(f"\n\n[StopIteration] in repeat_generator while retrieving {run_times} bytes.")
print(f"{' ' * 14}> Bytes are being written to ErrorFile_RepeaterBytes.txt...")
error_file = open("../ErrorFile_RepeaterBytes.txt", 'w')
error_file.write(helper.create_textual_hex(elements.hex(), space_distance=2, enter_distance=24))
error_file.close()
print(f"{' ' * 14}> Writing bytes finished.")
raise e
return elements
def _debug_write_from_source(self, filename, datatype, write_bytes=True):
"""This function is used as a test debugging writing. It writes parts of the read file to the filesystem."""
print("File writing from source started with attributes " + datatype +
"...")
file = open(filename, "wb" if write_bytes else "w")
selected_parts = []
for t in datatype:
if t == "f":
selected_parts.append(self._file)
elif t == "h":
selected_parts.append(self._file_header)
elif t == "d":
selected_parts.append(self._decompressed_file_data)
parts = None
for part in selected_parts:
if parts is None:
parts = part
continue
parts += part
file.write(parts if write_bytes else create_textual_hex(parts.hex()))
file.close()
print("File writing finished successfully.")
def get_byte_structure_as_string(self, skip_retrievers=None):
if skip_retrievers is None:
skip_retrievers = []
byte_structure = "\n" + self.get_header_string()
for retriever in self.retrievers:
if retriever.name in skip_retrievers:
continue
byte_structure += "\n"
listed_retriever_data = parser.listify(retriever.data)
struct_header_set = False
for struct in listed_retriever_data:
if isinstance(struct, AoE2Piece):
if not struct_header_set:
byte_structure += f"\n{'#' * 27} {retriever.name} ({retriever.datatype.to_simple_string()})"
struct_header_set = True
byte_structure += struct.get_byte_structure_as_string()
# Struct Header was set. Retriever was struct, data retrieved using recursion. Next retriever.
if struct_header_set:
byte_structure += f"{'#' * 27} End of: {retriever.name} ({retriever.datatype.to_simple_string()})\n"
continue
retriever_data_bytes = parser.retriever_to_bytes(retriever)
if retriever_data_bytes is None:
return byte_structure
else:
retriever_data_bytes = retriever_data_bytes.hex()
retriever_short_string = retriever.get_short_str()
retriever_data_hex = helper.create_textual_hex(
retriever_data_bytes, space_distance=2, enter_distance=24)
if "\n" in retriever_data_hex:
split_hex = retriever_data_hex.split("\n")
if "\r\n" in retriever_short_string:
split_data_string = retriever_short_string.split("\r\n")
split_hex_length = len(split_hex)
split_data_string_length = len(split_data_string)
lines = max(split_hex_length, split_data_string_length)
combined_strings = []
for i in range(0, lines):
combined_strings.append(
helper.add_suffix_chars(
split_hex[i] if i < split_hex_length else "",
" ", 28) +
(split_data_string[i]
if i < split_data_string_length else ""))
byte_structure += "\n".join(combined_strings)
else:
split_hex[0] = helper.add_suffix_chars(
split_hex[0], " ", 28) + retriever_short_string
byte_structure += "\n".join(split_hex)
else:
byte_structure += helper.add_suffix_chars(
retriever_data_hex, " ", 28) + retriever_short_string
return byte_structure + "\n"
def _debug_byte_structure_to_file(self,
filename,
generator_for_trail=None,
log_debug_write=True,
commit=False):
""" Used for debugging - Writes structure from read file to the filesystem in a easily readable manner. """
if commit and hasattr(self, '_object_manager'):
# self._object_manager.reconstruct(log_debug_write)
self._write_from_structure(filename,
log_writing=log_debug_write,
log_reconstructing=log_debug_write)
lgr = SimpleLogger(log_debug_write)
pieces = collections.OrderedDict(**self._parsed_header,
**self._parsed_data)
lgr.print("\nWriting structure to file...")
with open(filename, 'w', encoding="utf-8") as output_file:
result = []
for key in self._parsed_header:
lgr.print("\tWriting " + key + "...", replace_line=True)
result.append(
self._parsed_header[key].get_byte_structure_as_string(
pieces))
lgr.print("\tWriting " + key + " finished successfully.",
replace_line=True)
lgr.print()
for key in self._parsed_data:
lgr.print("\tWriting " + key + "...", replace_line=True)
result.append(
self._parsed_data[key].get_byte_structure_as_string(
pieces))
lgr.print("\tWriting " + key + " finished successfully.",
replace_line=True)
lgr.print()
if generator_for_trail is not None:
lgr.print("\tWriting trail...", replace_line=True)
trail_length = -1 # -1 == inf
try:
trail = b''
i = 0
while i != trail_length:
trail += generator.repeat_generator(
generator=generator_for_trail,
run_times=1,
intended_stop_iteration=True,
return_bytes=True)
i += 1
except StopIteration:
pass # Expected, if trail is not present or shorter than {trail_length} bytes
if i != 0:
i += 1
if i == trail_length:
i = str(i) + '+'
if trail_length == -1:
trail_length = i
result.append(f"\n\n{'#' * 27} TRAIL ({i}/{trail_length})\n\n")
result.append(
helper.create_textual_hex(trail.hex(),
space_distance=2,
enter_distance=24))
lgr.print("\tWriting trail finished successfully.",
replace_line=True)
lgr.print()
output_file.write(''.join(result))
output_file.close()
lgr.print("Writing structure to file finished successfully.")