def load_loader_hunk(f, program_data): program_data.loader_system_name = persistence.read_string(f) program_data.loader_segments = read_segment_list(f) program_data.loader_relocated_addresses = persistence.read_set_of_uint32s(f) program_data.loader_relocatable_addresses = persistence.read_set_of_uint32s(f) program_data.loader_entrypoint_segment_id = persistence.read_uint16(f) program_data.loader_entrypoint_offset = persistence.read_uint32(f) ## POST PROCESSING program_data.loader_data_types = loaderlib.get_system_data_types(program_data.loader_system_name)
def load_loader_hunk(f, program_data): program_data.loader_system_name = persistence.read_string(f) program_data.loader_segments = read_segment_list(f) program_data.loader_relocated_addresses = persistence.read_dict_uint32_to_set_of_uint32s( f) program_data.loader_relocatable_addresses = persistence.read_set_of_uint32s( f) program_data.loader_entrypoint_segment_id = persistence.read_uint16(f) program_data.loader_entrypoint_offset = persistence.read_uint32(f) ## POST PROCESSING program_data.loader_data_types = loaderlib.get_system_data_types( program_data.loader_system_name)
def read_SegmentBlock(f): block = SegmentBlock() bytes_to_read = struct.calcsize(SEGMENTBLOCK_PACK_FORMAT) block.segment_id, block.segment_offset, block.address, block.length, block.flags, block.line_count, line_data_count = struct.unpack(SEGMENTBLOCK_PACK_FORMAT, f.read(bytes_to_read)) if line_data_count > 0: if get_block_data_type(block) == DATA_TYPE_CODE: block.line_data = [ None ] * line_data_count for i in xrange(line_data_count): type_id = persistence.read_uint8(f) if type_id == SLD_INSTRUCTION: block_offset = persistence.read_uint16(f) block.line_data[i] = (type_id, block_offset) elif type_id == SLD_EQU_LOCATION_RELATIVE: block_offset = persistence.read_uint32(f) block.line_data[i] = (type_id, block_offset) elif type_id in (SLD_COMMENT_TRAILING, SLD_COMMENT_FULL_LINE): text = persistence.read_string(f) block.line_data[i] = (type_id, text) return block
def read_SegmentBlock(f): block = SegmentBlock() bytes_to_read = struct.calcsize(SEGMENTBLOCK_PACK_FORMAT) block.segment_id, block.segment_offset, block.address, block.length, block.flags, block.line_count, line_data_count = struct.unpack( SEGMENTBLOCK_PACK_FORMAT, f.read(bytes_to_read)) if line_data_count > 0: if get_block_data_type(block) == DATA_TYPE_CODE: block.line_data = [None] * line_data_count for i in range(line_data_count): type_id = persistence.read_uint8(f) if type_id == SLD_INSTRUCTION: block_offset = persistence.read_uint16(f) block.line_data[i] = (type_id, block_offset) elif type_id == SLD_EQU_LOCATION_RELATIVE: block_offset = persistence.read_uint32(f) block.line_data[i] = (type_id, block_offset) elif type_id in (SLD_COMMENT_TRAILING, SLD_COMMENT_FULL_LINE): text = persistence.read_string(f) block.line_data[i] = (type_id, text) return block
def convert_project_format_3_to_4(input_file): """ This function should encapsulate all application-specific logic involved to make it independent of as many changes as possible. From version: 3. To version: 4. Modifications: - disassembly hunk processor id has changed from string to uint32. """ SNAPSHOT_HUNK_VERSIONS = { SAVEFILE_HUNK_SOURCEDATA: 1, SAVEFILE_HUNK_SOURCEDATAINFO: 1, SAVEFILE_HUNK_LOADER: 1, SAVEFILE_HUNK_LOADERINTERNAL: 1, SAVEFILE_HUNK_DISASSEMBLY: 1, } input_file.seek(0, os.SEEK_END) file_size = input_file.tell() input_file.seek(0, os.SEEK_SET) savefile_id = persistence.read_uint32(input_file) savefile_version = persistence.read_uint16(input_file) if savefile_version != 3: return None logger.info("Upgrading save-file from version 3 to version 4: Processor id field..") save_count = persistence.read_uint32(input_file) output_file = tempfile.TemporaryFile() persistence.write_uint32(output_file, savefile_id) persistence.write_uint16(output_file, 4) persistence.write_uint32(output_file, save_count) while input_file.tell() < file_size: # This should be pretty straightforward. hunk_header_offset = input_file.tell() hunk_id = persistence.read_uint16(input_file) hunk_length = persistence.read_uint32(input_file) hunk_payload_offset = input_file.tell() actual_hunk_version = persistence.read_uint16(input_file) expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id] if expected_hunk_version != actual_hunk_version: logger.error("convert_project_format_3_to_4: hunk %d version mismatch %d != %d", hunk_id, expected_hunk_version, actual_hunk_version) return None logger.debug("convert_project_format_3_to_4: file hunk %d", hunk_id) # Copy unaffected hunks verbatim. if hunk_id != SAVEFILE_HUNK_DISASSEMBLY: input_file.seek(hunk_header_offset, os.SEEK_SET) raw_hunk_length = (hunk_payload_offset - hunk_header_offset) + hunk_length output_file.write(input_file.read(raw_hunk_length)) continue ## 1. Load the hunk payload. branch_addresses = persistence.read_dict_uint32_to_set_of_uint32s(input_file) reference_addresses = persistence.read_dict_uint32_to_set_of_uint32s(input_file) symbols_by_address = persistence.read_dict_uint32_to_string(input_file) post_segment_addresses = persistence.read_dict_uint32_to_list_of_uint32s(input_file) flags = persistence.read_uint32(input_file) processor_name = persistence.read_string(input_file) # Reconstitute the segment block list. num_blocks = persistence.read_uint32(input_file) input_file_offset = input_file.tell() block_data_length = hunk_length - (input_file_offset - hunk_payload_offset) block_data_string = input_file.read(block_data_length) #blocks = [ None ] * num_blocks #for i in xrange(num_blocks): # blocks[i] = read_SegmentBlock(input_file) ## 2. Write the generic hunk header, then the payload, then fill in the header. # Only these two are likely to have been in use. if processor_name == "m68k": processor_id = loaderlib.constants.PROCESSOR_M680x0 elif processor_name == "mips": processor_id = loaderlib.constants.PROCESSOR_MIPS else: logger.error("convert_project_format_3_to_4: unrecognised arch name %s", processor_name) return None logger.debug("convert_project_format_3_to_4: arch name %s maps to processor id %d", processor_name, processor_id) # Write the as yet to be updated header. persistence.write_uint16(output_file, hunk_id) output_file_length_offset = output_file.tell() persistence.write_uint32(output_file, 0) output_file_payload_offset = output_file.tell() persistence.write_uint16(output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_DISASSEMBLY] + 1) # Write the payload in the modified format. persistence.write_dict_uint32_to_set_of_uint32s(output_file, branch_addresses) persistence.write_dict_uint32_to_set_of_uint32s(output_file, reference_addresses) persistence.write_dict_uint32_to_string(output_file, symbols_by_address) persistence.write_dict_uint32_to_list_of_uint32s(output_file, post_segment_addresses) persistence.write_uint32(output_file, flags) persistence.write_uint32(output_file, processor_id) persistence.write_uint32(output_file, num_blocks) output_file_offset = output_file.tell() #for block in blocks: # write_SegmentBlock(output_file, block) output_file.write(block_data_string) if output_file.tell() - output_file_offset != block_data_length: logger.error("convert_project_format_3_to_4: block length mismatch %d != %d", output_file.tell() - output_file_offset, block_data_length) return None # Update the header length field, then fast forward to the end of the hunk. new_hunk_length = output_file.tell() - output_file_payload_offset output_file.seek(output_file_length_offset, os.SEEK_SET) persistence.write_uint32(output_file, new_hunk_length) output_file.seek(new_hunk_length, os.SEEK_CUR) if output_file.tell() - output_file_payload_offset != new_hunk_length: logger.error("convert_project_format_3_to_4: block length mismatch %d != %d", output_file.tell() - output_file_payload_offset, new_hunk_length) return None return output_file
def convert_project_format_4_to_5(input_file): """ This function should encapsulate all application-specific logic involved to make it independent of as many changes as possible. Version 4 -> 5. Modifications: - symbols by address has gone from a label to a structure with metadata. """ SNAPSHOT_HUNK_VERSIONS = { SAVEFILE_HUNK_SOURCEDATA: 1, SAVEFILE_HUNK_SOURCEDATAINFO: 1, SAVEFILE_HUNK_LOADER: 1, SAVEFILE_HUNK_LOADERINTERNAL: 1, SAVEFILE_HUNK_DISASSEMBLY: 2, } input_file.seek(0, os.SEEK_END) file_size = input_file.tell() input_file.seek(0, os.SEEK_SET) savefile_id = persistence.read_uint32(input_file) savefile_version = persistence.read_uint16(input_file) if savefile_version != 4: return None logger.info( "Upgrading save-file from version 4 to version 5: symbols by address.." ) save_count = persistence.read_uint32(input_file) output_file = tempfile.TemporaryFile() persistence.write_uint32(output_file, savefile_id) persistence.write_uint16(output_file, 5) persistence.write_uint32(output_file, save_count) while input_file.tell() < file_size: # This should be pretty straightforward. hunk_header_offset = input_file.tell() hunk_id = persistence.read_uint16(input_file) hunk_length = persistence.read_uint32(input_file) hunk_payload_offset = input_file.tell() actual_hunk_version = persistence.read_uint16(input_file) expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id] if expected_hunk_version != actual_hunk_version: logger.error( "convert_project_format_4_to_5: hunk %d version mismatch %d != %d", hunk_id, expected_hunk_version, actual_hunk_version) return None logger.debug("convert_project_format_4_to_5: file hunk %d", hunk_id) if SAVEFILE_HUNK_LOADER != hunk_id: input_file.seek(hunk_header_offset, os.SEEK_SET) raw_hunk_length = (hunk_payload_offset - hunk_header_offset) + hunk_length output_file.write(input_file.read(raw_hunk_length)) continue # Write the as yet to be updated header. persistence.write_uint16(output_file, hunk_id) output_file_length_offset = output_file.tell() persistence.write_uint32(output_file, 0) output_file_payload_offset = output_file.tell() persistence.write_uint16( output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_LOADER] + 1) # Transform the hunk from input file to output file. persistence.write_string( output_file, persistence.read_string(input_file)) # loader_system_name data_size = persistence.read_uint32(input_file) persistence.write_uint32(output_file, data_size) # loader_segments output_file.write(input_file.read(data_size)) set_value = persistence.read_set_of_uint32s(input_file) persistence.write_dict_uint32_to_list_of_uint32s( output_file, {k: [] for k in set_value}) # loader_relocated_addresses set_value = persistence.read_set_of_uint32s(input_file) persistence.write_set_of_uint32s( output_file, set_value) # loader_relocatable_addresses persistence.write_uint16( output_file, persistence.read_uint16( input_file)) # loader_entrypoint_segment_id persistence.write_int32( output_file, persistence.read_uint32(input_file)) # loader_entrypoint_offset # Update the header length field, then fast forward to the end of the hunk. new_hunk_length = output_file.tell() - output_file_payload_offset output_file.seek(output_file_length_offset, os.SEEK_SET) persistence.write_uint32(output_file, new_hunk_length) output_file.seek(new_hunk_length, os.SEEK_CUR) if output_file.tell() - output_file_payload_offset != new_hunk_length: logger.error( "convert_project_format_4_to_5: block length mismatch %d != %d", output_file.tell() - output_file_payload_offset, new_hunk_length) return None return output_file
def convert_project_format_3_to_4(input_file): """ This function should encapsulate all application-specific logic involved to make it independent of as many changes as possible. From version: 3. To version: 4. Modifications: - disassembly hunk processor id has changed from string to uint32. """ SNAPSHOT_HUNK_VERSIONS = { SAVEFILE_HUNK_SOURCEDATA: 1, SAVEFILE_HUNK_SOURCEDATAINFO: 1, SAVEFILE_HUNK_LOADER: 1, SAVEFILE_HUNK_LOADERINTERNAL: 1, SAVEFILE_HUNK_DISASSEMBLY: 1, } input_file.seek(0, os.SEEK_END) file_size = input_file.tell() input_file.seek(0, os.SEEK_SET) savefile_id = persistence.read_uint32(input_file) savefile_version = persistence.read_uint16(input_file) if savefile_version != 3: return None logger.info( "Upgrading save-file from version 3 to version 4: Processor id field.." ) save_count = persistence.read_uint32(input_file) output_file = tempfile.TemporaryFile() persistence.write_uint32(output_file, savefile_id) persistence.write_uint16(output_file, 4) persistence.write_uint32(output_file, save_count) while input_file.tell() < file_size: # This should be pretty straightforward. hunk_header_offset = input_file.tell() hunk_id = persistence.read_uint16(input_file) hunk_length = persistence.read_uint32(input_file) hunk_payload_offset = input_file.tell() actual_hunk_version = persistence.read_uint16(input_file) expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id] if expected_hunk_version != actual_hunk_version: logger.error( "convert_project_format_3_to_4: hunk %d version mismatch %d != %d", hunk_id, expected_hunk_version, actual_hunk_version) return None logger.debug("convert_project_format_3_to_4: file hunk %d", hunk_id) # Copy unaffected hunks verbatim. if hunk_id != SAVEFILE_HUNK_DISASSEMBLY: input_file.seek(hunk_header_offset, os.SEEK_SET) raw_hunk_length = (hunk_payload_offset - hunk_header_offset) + hunk_length output_file.write(input_file.read(raw_hunk_length)) continue ## 1. Load the hunk payload. branch_addresses = persistence.read_dict_uint32_to_set_of_uint32s( input_file) reference_addresses = persistence.read_dict_uint32_to_set_of_uint32s( input_file) symbols_by_address = persistence.read_dict_uint32_to_string(input_file) post_segment_addresses = persistence.read_dict_uint32_to_list_of_uint32s( input_file) flags = persistence.read_uint32(input_file) processor_name = persistence.read_string(input_file) # Reconstitute the segment block list. num_blocks = persistence.read_uint32(input_file) input_file_offset = input_file.tell() block_data_length = hunk_length - (input_file_offset - hunk_payload_offset) block_data_string = input_file.read(block_data_length) #blocks = [ None ] * num_blocks #for i in range(num_blocks): # blocks[i] = read_SegmentBlock(input_file) ## 2. Write the generic hunk header, then the payload, then fill in the header. # Only these two are likely to have been in use. if processor_name == "m68k": processor_id = loaderlib.constants.PROCESSOR_M680x0 elif processor_name == "mips": processor_id = loaderlib.constants.PROCESSOR_MIPS else: logger.error( "convert_project_format_3_to_4: unrecognised arch name %s", processor_name) return None logger.debug( "convert_project_format_3_to_4: arch name %s maps to processor id %d", processor_name, processor_id) # Write the as yet to be updated header. persistence.write_uint16(output_file, hunk_id) output_file_length_offset = output_file.tell() persistence.write_uint32(output_file, 0) output_file_payload_offset = output_file.tell() persistence.write_uint16( output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_DISASSEMBLY] + 1) # Write the payload in the modified format. persistence.write_dict_uint32_to_set_of_uint32s( output_file, branch_addresses) persistence.write_dict_uint32_to_set_of_uint32s( output_file, reference_addresses) persistence.write_dict_uint32_to_string(output_file, symbols_by_address) persistence.write_dict_uint32_to_list_of_uint32s( output_file, post_segment_addresses) persistence.write_uint32(output_file, flags) persistence.write_uint32(output_file, processor_id) persistence.write_uint32(output_file, num_blocks) output_file_offset = output_file.tell() #for block in blocks: # write_SegmentBlock(output_file, block, program_data.dis_constant_pc_offset) output_file.write(block_data_string) if output_file.tell() - output_file_offset != block_data_length: logger.error( "convert_project_format_3_to_4: block length mismatch %d != %d", output_file.tell() - output_file_offset, block_data_length) return None # Update the header length field, then fast forward to the end of the hunk. new_hunk_length = output_file.tell() - output_file_payload_offset output_file.seek(output_file_length_offset, os.SEEK_SET) persistence.write_uint32(output_file, new_hunk_length) output_file.seek(new_hunk_length, os.SEEK_CUR) if output_file.tell() - output_file_payload_offset != new_hunk_length: logger.error( "convert_project_format_3_to_4: block length mismatch %d != %d", output_file.tell() - output_file_payload_offset, new_hunk_length) return None return output_file
def convert_project_format_4_to_5(input_file): """ This function should encapsulate all application-specific logic involved to make it independent of as many changes as possible. Version 4 -> 5. Modifications: - symbols by address has gone from a label to a structure with metadata. """ SNAPSHOT_HUNK_VERSIONS = { SAVEFILE_HUNK_SOURCEDATA: 1, SAVEFILE_HUNK_SOURCEDATAINFO: 1, SAVEFILE_HUNK_LOADER: 1, SAVEFILE_HUNK_LOADERINTERNAL: 1, SAVEFILE_HUNK_DISASSEMBLY: 2, } input_file.seek(0, os.SEEK_END) file_size = input_file.tell() input_file.seek(0, os.SEEK_SET) savefile_id = persistence.read_uint32(input_file) savefile_version = persistence.read_uint16(input_file) if savefile_version != 4: return None logger.info("Upgrading save-file from version 4 to version 5: symbols by address..") save_count = persistence.read_uint32(input_file) output_file = tempfile.TemporaryFile() persistence.write_uint32(output_file, savefile_id) persistence.write_uint16(output_file, 5) persistence.write_uint32(output_file, save_count) while input_file.tell() < file_size: # This should be pretty straightforward. hunk_header_offset = input_file.tell() hunk_id = persistence.read_uint16(input_file) hunk_length = persistence.read_uint32(input_file) hunk_payload_offset = input_file.tell() actual_hunk_version = persistence.read_uint16(input_file) expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id] if expected_hunk_version != actual_hunk_version: logger.error("convert_project_format_4_to_5: hunk %d version mismatch %d != %d", hunk_id, expected_hunk_version, actual_hunk_version) return None logger.debug("convert_project_format_4_to_5: file hunk %d", hunk_id) if SAVEFILE_HUNK_LOADER != hunk_id: input_file.seek(hunk_header_offset, os.SEEK_SET) raw_hunk_length = (hunk_payload_offset - hunk_header_offset) + hunk_length output_file.write(input_file.read(raw_hunk_length)) continue # Write the as yet to be updated header. persistence.write_uint16(output_file, hunk_id) output_file_length_offset = output_file.tell() persistence.write_uint32(output_file, 0) output_file_payload_offset = output_file.tell() persistence.write_uint16(output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_LOADER] + 1) # Transform the hunk from input file to output file. persistence.write_string(output_file, persistence.read_string(input_file)) # loader_system_name data_size = persistence.read_uint32(input_file) persistence.write_uint32(output_file, data_size) # loader_segments output_file.write(input_file.read(data_size)) set_value = persistence.read_set_of_uint32s(input_file) persistence.write_dict_uint32_to_list_of_uint32s(output_file, { k: [] for k in set_value }) # loader_relocated_addresses set_value = persistence.read_set_of_uint32s(input_file) persistence.write_set_of_uint32s(output_file, set_value) # loader_relocatable_addresses persistence.write_uint16(output_file, persistence.read_uint16(input_file)) # loader_entrypoint_segment_id persistence.write_int32(output_file, persistence.read_uint32(input_file)) # loader_entrypoint_offset # Update the header length field, then fast forward to the end of the hunk. new_hunk_length = output_file.tell() - output_file_payload_offset output_file.seek(output_file_length_offset, os.SEEK_SET) persistence.write_uint32(output_file, new_hunk_length) output_file.seek(new_hunk_length, os.SEEK_CUR) if output_file.tell() - output_file_payload_offset != new_hunk_length: logger.error("convert_project_format_4_to_5: block length mismatch %d != %d", output_file.tell() - output_file_payload_offset, new_hunk_length) return None return output_file