def _write_from_structure(self, filename, write_in_bytes=True, compress=True, log_writing=True, log_reconstructing=False): if hasattr(self, '_object_manager'): self._object_manager.reconstruct( log_reconstructing=log_reconstructing) lgr = SimpleLogger(log_writing) lgr.print("\nFile writing from structure started...") byte_header_list = [] byte_data_list = [] for key in self._parsed_header: lgr.print("\twriting " + key + "...", replace_line=True) for retriever in self._parsed_header[key].retrievers: byte_header_list.append(parser.retriever_to_bytes(retriever)) lgr.print("\twriting " + key + " finished successfully.", replace_line=True) lgr.print() for key in self._parsed_data: lgr.print("\twriting " + key + "...", replace_line=True) for retriever in self._parsed_data[key].retrievers: try: byte_data_list.append(parser.retriever_to_bytes(retriever)) except AttributeError as e: print("AttributeError occurred while writing '" + key + "' > '" + retriever.name + "'") print("\n\n\nAn error occurred. Writing failed.") raise e lgr.print("\twriting " + key + " finished successfully.", replace_line=True) lgr.print() file = open(filename, "wb" if write_in_bytes else "w") byte_header = b''.join(byte_header_list) byte_data = b''.join(byte_data_list) file.write(byte_header if write_in_bytes else create_textual_hex( byte_header.hex())) if compress: lgr.print("\tCompressing...", replace_line=True) # https://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760 deflate_obj = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS) compressed = deflate_obj.compress( b''.join(byte_data_list)) + deflate_obj.flush() file.write(compressed if write_in_bytes else create_textual_hex( compressed.hex())) lgr.print("\tCompressing finished successfully.", replace_line=True) lgr.print() else: file.write(byte_data if write_in_bytes else create_textual_hex( byte_data.hex())) file.close() lgr.print("File writing finished successfully.")
def get_byte_structure_as_string(self, skip_retrievers=None): if skip_retrievers is None: skip_retrievers = [] byte_structure = "\n" + self.get_header_string() for retriever in self.retrievers: if retriever.name in skip_retrievers: continue byte_structure += "\n" if type(retriever.data) is list and len(retriever.data) > 0: if isinstance(retriever.data[0], AoE2Piece): for struct in retriever.data: byte_structure += struct.get_byte_structure_as_string() continue else: if isinstance(retriever.data, AoE2Piece): byte_structure += retriever.data.get_byte_structure_as_string( ) continue retriever_data_bytes = parser.retriever_to_bytes(retriever) if retriever_data_bytes is None: return byte_structure else: retriever_data_bytes = retriever_data_bytes.hex() retriever_short_string = retriever.get_short_str() retriever_data_hex = helper.create_textual_hex( retriever_data_bytes, space_distance=2, enter_distance=24) if "\n" in retriever_data_hex: split_hex = retriever_data_hex.split("\n") if "\r\n" in retriever_short_string: split_data_string = retriever_short_string.split("\r\n") split_hex_length = len(split_hex) split_data_string_length = len(split_data_string) lines = max(split_hex_length, split_data_string_length) combined_strings = [] for i in range(0, lines): combined_strings.append( helper.add_suffix_chars( split_hex[i] if i < split_hex_length else "", " ", 28) + (split_data_string[i] if i < split_data_string_length else "")) byte_structure += "\n".join(combined_strings) else: split_hex[0] = helper.add_suffix_chars( split_hex[0], " ", 28) + retriever_short_string byte_structure += "\n".join(split_hex) else: byte_structure += helper.add_suffix_chars( retriever_data_hex, " ", 28) + retriever_short_string return byte_structure + "\n\n"
def get_byte_structure_as_string(self, pieces, skip_retrievers=None): if skip_retrievers is None: skip_retrievers = [] byte_structure = "\n" + self.get_header_string() for retriever in self.retrievers: if retriever.name in skip_retrievers: continue byte_structure += "\n" listed_retriever_data = parser.listify(retriever.data) struct_header_set = False for struct in listed_retriever_data: if isinstance(struct, AoE2Piece): if not struct_header_set: byte_structure += f"\n{'#' * 27} {retriever.name} ({retriever.datatype.to_simple_string()})" struct_header_set = True byte_structure += struct.get_byte_structure_as_string(pieces) # Struct Header was set. Retriever was struct, data retrieved using recursion. Next retriever. if struct_header_set: byte_structure += f"{'#' * 27} End of: {retriever.name} ({retriever.datatype.to_simple_string()})\n" continue retriever_data_bytes = parser.retriever_to_bytes(retriever, pieces) if retriever_data_bytes is None: return byte_structure else: retriever_data_bytes = retriever_data_bytes.hex() retriever_short_string: str = retriever.get_short_str() retriever_hex = helper.create_textual_hex( retriever_data_bytes, space_distance=2, enter_distance=24 ) split_hex = retriever_hex.split("\n") split_hex_length = len(split_hex) split_data_string = retriever_short_string.replace('\x00', ' ').splitlines() data_lines = [] for x in split_data_string: if len(x) > 120: data_lines += [f'\t{x}' for x in helper.insert_char(x, '\r\n', 120).splitlines()] else: data_lines.append(x) split_data_length = len(data_lines) lines = max(split_hex_length, split_data_length) combined_strings = [] for i in range(0, lines): hex_part = split_hex[i] if i < split_hex_length else "" data_part = data_lines[i] if i < split_data_length else "" combined_strings.append(helper.add_suffix_chars(hex_part, " ", 28) + data_part) byte_structure += "\n".join(combined_strings) return byte_structure + "\n"
def _debug_write_from_source(self, filename, datatype, write_in_bytes=True): """This function is used as a test debugging writing. It writes parts of the read file to the filesystem.""" print("File writing from source started with attributes " + datatype + "...") file = open(filename, "wb" if write_in_bytes else "w") for t in datatype: if t == "f": file.write(self._file if write_in_bytes else create_textual_hex(self._file.hex())) elif t == "h": file.write(self._file_header if write_in_bytes else create_textual_hex(self._file_header.hex())) elif t == "d": file.write(self._file_data if write_in_bytes else create_textual_hex(self._file_data.hex())) file.close() print("File writing finished successfully.")
def repeat_generator(generator, run_times, intended_stop_iteration=False, return_bytes=True): elements = b'' if return_bytes else [] try: for i in range(0, run_times): if return_bytes: elements += next(generator) else: # Generator returns list. So lists are merged elements += next(generator) except StopIteration as e: if not intended_stop_iteration: print(f"\n\n[StopIteration] in repeat_generator while retrieving {run_times} bytes.") print(f"{' ' * 14}> Bytes are being written to ErrorFile_RepeaterBytes.txt...") error_file = open("../ErrorFile_RepeaterBytes.txt", 'w') error_file.write(helper.create_textual_hex(elements.hex(), space_distance=2, enter_distance=24)) error_file.close() print(f"{' ' * 14}> Writing bytes finished.") raise e return elements
def _debug_write_from_source(self, filename, datatype, write_bytes=True): """This function is used as a test debugging writing. It writes parts of the read file to the filesystem.""" print("File writing from source started with attributes " + datatype + "...") file = open(filename, "wb" if write_bytes else "w") selected_parts = [] for t in datatype: if t == "f": selected_parts.append(self._file) elif t == "h": selected_parts.append(self._file_header) elif t == "d": selected_parts.append(self._decompressed_file_data) parts = None for part in selected_parts: if parts is None: parts = part continue parts += part file.write(parts if write_bytes else create_textual_hex(parts.hex())) file.close() print("File writing finished successfully.")
def get_byte_structure_as_string(self, skip_retrievers=None): if skip_retrievers is None: skip_retrievers = [] byte_structure = "\n" + self.get_header_string() for retriever in self.retrievers: if retriever.name in skip_retrievers: continue byte_structure += "\n" listed_retriever_data = parser.listify(retriever.data) struct_header_set = False for struct in listed_retriever_data: if isinstance(struct, AoE2Piece): if not struct_header_set: byte_structure += f"\n{'#' * 27} {retriever.name} ({retriever.datatype.to_simple_string()})" struct_header_set = True byte_structure += struct.get_byte_structure_as_string() # Struct Header was set. Retriever was struct, data retrieved using recursion. Next retriever. if struct_header_set: byte_structure += f"{'#' * 27} End of: {retriever.name} ({retriever.datatype.to_simple_string()})\n" continue retriever_data_bytes = parser.retriever_to_bytes(retriever) if retriever_data_bytes is None: return byte_structure else: retriever_data_bytes = retriever_data_bytes.hex() retriever_short_string = retriever.get_short_str() retriever_data_hex = helper.create_textual_hex( retriever_data_bytes, space_distance=2, enter_distance=24) if "\n" in retriever_data_hex: split_hex = retriever_data_hex.split("\n") if "\r\n" in retriever_short_string: split_data_string = retriever_short_string.split("\r\n") split_hex_length = len(split_hex) split_data_string_length = len(split_data_string) lines = max(split_hex_length, split_data_string_length) combined_strings = [] for i in range(0, lines): combined_strings.append( helper.add_suffix_chars( split_hex[i] if i < split_hex_length else "", " ", 28) + (split_data_string[i] if i < split_data_string_length else "")) byte_structure += "\n".join(combined_strings) else: split_hex[0] = helper.add_suffix_chars( split_hex[0], " ", 28) + retriever_short_string byte_structure += "\n".join(split_hex) else: byte_structure += helper.add_suffix_chars( retriever_data_hex, " ", 28) + retriever_short_string return byte_structure + "\n"
def _debug_byte_structure_to_file(self, filename, generator_for_trail=None, log_debug_write=True, commit=False): """ Used for debugging - Writes structure from read file to the filesystem in a easily readable manner. """ if commit and hasattr(self, '_object_manager'): # self._object_manager.reconstruct(log_debug_write) self._write_from_structure(filename, log_writing=log_debug_write, log_reconstructing=log_debug_write) lgr = SimpleLogger(log_debug_write) pieces = collections.OrderedDict(**self._parsed_header, **self._parsed_data) lgr.print("\nWriting structure to file...") with open(filename, 'w', encoding="utf-8") as output_file: result = [] for key in self._parsed_header: lgr.print("\tWriting " + key + "...", replace_line=True) result.append( self._parsed_header[key].get_byte_structure_as_string( pieces)) lgr.print("\tWriting " + key + " finished successfully.", replace_line=True) lgr.print() for key in self._parsed_data: lgr.print("\tWriting " + key + "...", replace_line=True) result.append( self._parsed_data[key].get_byte_structure_as_string( pieces)) lgr.print("\tWriting " + key + " finished successfully.", replace_line=True) lgr.print() if generator_for_trail is not None: lgr.print("\tWriting trail...", replace_line=True) trail_length = -1 # -1 == inf try: trail = b'' i = 0 while i != trail_length: trail += generator.repeat_generator( generator=generator_for_trail, run_times=1, intended_stop_iteration=True, return_bytes=True) i += 1 except StopIteration: pass # Expected, if trail is not present or shorter than {trail_length} bytes if i != 0: i += 1 if i == trail_length: i = str(i) + '+' if trail_length == -1: trail_length = i result.append(f"\n\n{'#' * 27} TRAIL ({i}/{trail_length})\n\n") result.append( helper.create_textual_hex(trail.hex(), space_distance=2, enter_distance=24)) lgr.print("\tWriting trail finished successfully.", replace_line=True) lgr.print() output_file.write(''.join(result)) output_file.close() lgr.print("Writing structure to file finished successfully.")