def convert_project_format_2_to_3(input_file):
"""
This function should encapsulate all application-specific logic involved to
make it independent of as many changes as possible.
From version: 2.
To version: 3.
Modifications:
- Inserts a version number into all hunks.
"""
SNAPSHOT_HUNK_VERSIONS = {
SAVEFILE_HUNK_SOURCEDATA: 1,
SAVEFILE_HUNK_SOURCEDATAINFO: 1,
SAVEFILE_HUNK_LOADER: 1,
SAVEFILE_HUNK_LOADERINTERNAL: 1,
SAVEFILE_HUNK_DISASSEMBLY: 1,
}
input_file.seek(0, os.SEEK_END)
file_size = input_file.tell()
input_file.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(input_file)
savefile_version = persistence.read_uint16(input_file)
if savefile_version != 2:
return None
logger.info(
"Upgrading save-file from version 2 to version 3: Hunk versioning..")
save_count = persistence.read_uint32(input_file)
output_file = tempfile.TemporaryFile()
persistence.write_uint32(output_file, savefile_id)
persistence.write_uint16(output_file, 3)
persistence.write_uint32(output_file, save_count)
while input_file.tell() < file_size:
# This should be pretty straightforward.
hunk_id = persistence.read_uint16(input_file)
persistence.write_uint16(output_file, hunk_id)
input_hunk_length = persistence.read_uint32(input_file)
output_length_offset = output_file.tell()
persistence.write_uint32(output_file, 0)
output_data_offset = output_file.tell()
# Modification.
persistence.write_uint16(output_file, SNAPSHOT_HUNK_VERSIONS[hunk_id])
input_data = input_file.read(input_hunk_length)
output_file.write(input_data)
output_hunk_length = output_file.tell() - output_data_offset
output_file.seek(output_length_offset, os.SEEK_SET)
persistence.write_uint32(output_file, output_hunk_length)
output_file.seek(output_hunk_length, os.SEEK_CUR)
return output_file
def convert_project_format_2_to_3(input_file):
"""
This function should encapsulate all application-specific logic involved to
make it independent of as many changes as possible.
From version: 2.
To version: 3.
Modifications:
- Inserts a version number into all hunks.
"""
SNAPSHOT_HUNK_VERSIONS = {
SAVEFILE_HUNK_SOURCEDATA: 1,
SAVEFILE_HUNK_SOURCEDATAINFO: 1,
SAVEFILE_HUNK_LOADER: 1,
SAVEFILE_HUNK_LOADERINTERNAL: 1,
SAVEFILE_HUNK_DISASSEMBLY: 1,
}
input_file.seek(0, os.SEEK_END)
file_size = input_file.tell()
input_file.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(input_file)
savefile_version = persistence.read_uint16(input_file)
if savefile_version != 2:
return None
logger.info("Upgrading save-file from version 2 to version 3: Hunk versioning..")
save_count = persistence.read_uint32(input_file)
output_file = tempfile.TemporaryFile()
persistence.write_uint32(output_file, savefile_id)
persistence.write_uint16(output_file, 3)
persistence.write_uint32(output_file, save_count)
while input_file.tell() < file_size:
# This should be pretty straightforward.
hunk_id = persistence.read_uint16(input_file)
persistence.write_uint16(output_file, hunk_id)
input_hunk_length = persistence.read_uint32(input_file)
output_length_offset = output_file.tell()
persistence.write_uint32(output_file, 0)
output_data_offset = output_file.tell()
# Modification.
persistence.write_uint16(output_file, SNAPSHOT_HUNK_VERSIONS[hunk_id])
input_data = input_file.read(input_hunk_length)
output_file.write(input_data)
output_hunk_length = output_file.tell() - output_data_offset
output_file.seek(output_length_offset, os.SEEK_SET)
persistence.write_uint32(output_file, output_hunk_length)
output_file.seek(output_hunk_length, os.SEEK_CUR)
return output_file
def read_segment_list_entry(f):
v = [ None ] * loaderlib.SIZEOF_SI
v[loaderlib.SI_TYPE] = persistence.read_uint8(f)
offset_value = persistence.read_uint32(f)
if offset_value == 0xFFFFFFFF: # Unsigned, special value.
offset_value = -1
v[loaderlib.SI_FILE_OFFSET] = offset_value
v[loaderlib.SI_DATA_LENGTH] = persistence.read_uint32(f)
v[loaderlib.SI_LENGTH] = persistence.read_uint32(f)
v[loaderlib.SI_ADDRESS] = persistence.read_uint32(f)
return v
def read_segment_list_entry(f):
v = [None] * loaderlib.SIZEOF_SI
v[loaderlib.SI_TYPE] = persistence.read_uint8(f)
offset_value = persistence.read_uint32(f)
if offset_value == 0xFFFFFFFF: # Unsigned, special value.
offset_value = -1
v[loaderlib.SI_FILE_OFFSET] = offset_value
v[loaderlib.SI_DATA_LENGTH] = persistence.read_uint32(f)
v[loaderlib.SI_LENGTH] = persistence.read_uint32(f)
v[loaderlib.SI_ADDRESS] = persistence.read_uint32(f)
return v
def read_segment_list(f):
num_bytes = persistence.read_uint32(f)
data_start_offset = f.tell()
v = []
while f.tell() - data_start_offset != num_bytes:
v.append(read_segment_list_entry(f))
return v
def read_segment_list(f):
num_bytes = persistence.read_uint32(f)
data_start_offset = f.tell()
v = []
while f.tell() - data_start_offset != num_bytes:
v.append(read_segment_list_entry(f))
return v
def load_loader_hunk(f, program_data):
program_data.loader_system_name = persistence.read_string(f)
program_data.loader_segments = read_segment_list(f)
program_data.loader_relocated_addresses = persistence.read_set_of_uint32s(f)
program_data.loader_relocatable_addresses = persistence.read_set_of_uint32s(f)
program_data.loader_entrypoint_segment_id = persistence.read_uint16(f)
program_data.loader_entrypoint_offset = persistence.read_uint32(f)
## POST PROCESSING
program_data.loader_data_types = loaderlib.get_system_data_types(program_data.loader_system_name)
def load_disassembly_hunk(f, program_data):
program_data.branch_addresses = persistence.read_dict_uint32_to_set_of_uint32s(f)
program_data.reference_addresses = persistence.read_dict_uint32_to_set_of_uint32s(f)
program_data.symbols_by_address = persistence.read_dict_uint32_to_string(f)
program_data.post_segment_addresses = persistence.read_dict_uint32_to_list_of_uint32s(f)
program_data.flags = persistence.read_uint32(f)
program_data.processor_id = persistence.read_uint32(f)
# Reconstitute the segment block list.
num_blocks = persistence.read_uint32(f)
program_data.blocks = [ None ] * num_blocks
for i in xrange(num_blocks):
program_data.blocks[i] = read_SegmentBlock(f)
## POST PROCESSING
# Rebuild the segment block list indexing lists.
program_data.block_addresses = [ 0 ] * num_blocks
program_data.block_line0s_dirtyidx = 0
program_data.block_line0s = program_data.block_addresses[:]
for i in xrange(num_blocks):
program_data.block_addresses[i] = program_data.blocks[i].address
def load_loader_hunk(f, program_data):
program_data.loader_system_name = persistence.read_string(f)
program_data.loader_segments = read_segment_list(f)
program_data.loader_relocated_addresses = persistence.read_dict_uint32_to_set_of_uint32s(
f)
program_data.loader_relocatable_addresses = persistence.read_set_of_uint32s(
f)
program_data.loader_entrypoint_segment_id = persistence.read_uint16(f)
program_data.loader_entrypoint_offset = persistence.read_uint32(f)
## POST PROCESSING
program_data.loader_data_types = loaderlib.get_system_data_types(
program_data.loader_system_name)
def load_disassembly_hunk(f, program_data):
program_data.branch_addresses = persistence.read_dict_uint32_to_set_of_uint32s(
f)
program_data.reference_addresses = persistence.read_dict_uint32_to_set_of_uint32s(
f)
program_data.symbols_by_address = persistence.read_dict_uint32_to_string(f)
program_data.post_segment_addresses = persistence.read_dict_uint32_to_list_of_uint32s(
f)
program_data.flags = persistence.read_uint32(f)
program_data.processor_id = persistence.read_uint32(f)
# Reconstitute the segment block list.
num_blocks = persistence.read_uint32(f)
program_data.blocks = [None] * num_blocks
for i in range(num_blocks):
program_data.blocks[i] = read_SegmentBlock(f)
## POST PROCESSING
# Rebuild the segment block list indexing lists.
program_data.block_addresses = [0] * num_blocks
program_data.block_line0s_dirtyidx = 0
program_data.block_line0s = program_data.block_addresses[:]
for i in range(num_blocks):
program_data.block_addresses[i] = program_data.blocks[i].address
def read_SegmentBlock(f):
block = SegmentBlock()
bytes_to_read = struct.calcsize(SEGMENTBLOCK_PACK_FORMAT)
block.segment_id, block.segment_offset, block.address, block.length, block.flags, block.line_count, line_data_count = struct.unpack(SEGMENTBLOCK_PACK_FORMAT, f.read(bytes_to_read))
if line_data_count > 0:
if get_block_data_type(block) == DATA_TYPE_CODE:
block.line_data = [ None ] * line_data_count
for i in xrange(line_data_count):
type_id = persistence.read_uint8(f)
if type_id == SLD_INSTRUCTION:
block_offset = persistence.read_uint16(f)
block.line_data[i] = (type_id, block_offset)
elif type_id == SLD_EQU_LOCATION_RELATIVE:
block_offset = persistence.read_uint32(f)
block.line_data[i] = (type_id, block_offset)
elif type_id in (SLD_COMMENT_TRAILING, SLD_COMMENT_FULL_LINE):
text = persistence.read_string(f)
block.line_data[i] = (type_id, text)
return block
def read_SegmentBlock(f):
block = SegmentBlock()
bytes_to_read = struct.calcsize(SEGMENTBLOCK_PACK_FORMAT)
block.segment_id, block.segment_offset, block.address, block.length, block.flags, block.line_count, line_data_count = struct.unpack(
SEGMENTBLOCK_PACK_FORMAT, f.read(bytes_to_read))
if line_data_count > 0:
if get_block_data_type(block) == DATA_TYPE_CODE:
block.line_data = [None] * line_data_count
for i in range(line_data_count):
type_id = persistence.read_uint8(f)
if type_id == SLD_INSTRUCTION:
block_offset = persistence.read_uint16(f)
block.line_data[i] = (type_id, block_offset)
elif type_id == SLD_EQU_LOCATION_RELATIVE:
block_offset = persistence.read_uint32(f)
block.line_data[i] = (type_id, block_offset)
elif type_id in (SLD_COMMENT_TRAILING, SLD_COMMENT_FULL_LINE):
text = persistence.read_string(f)
block.line_data[i] = (type_id, text)
return block
def convert_project_format_4_to_5(input_file):
"""
This function should encapsulate all application-specific logic involved to
make it independent of as many changes as possible.
Version 4 -> 5.
Modifications:
- symbols by address has gone from a label to a structure with metadata.
"""
SNAPSHOT_HUNK_VERSIONS = {
SAVEFILE_HUNK_SOURCEDATA: 1,
SAVEFILE_HUNK_SOURCEDATAINFO: 1,
SAVEFILE_HUNK_LOADER: 1,
SAVEFILE_HUNK_LOADERINTERNAL: 1,
SAVEFILE_HUNK_DISASSEMBLY: 2,
}
input_file.seek(0, os.SEEK_END)
file_size = input_file.tell()
input_file.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(input_file)
savefile_version = persistence.read_uint16(input_file)
if savefile_version != 4:
return None
logger.info("Upgrading save-file from version 4 to version 5: symbols by address..")
save_count = persistence.read_uint32(input_file)
output_file = tempfile.TemporaryFile()
persistence.write_uint32(output_file, savefile_id)
persistence.write_uint16(output_file, 5)
persistence.write_uint32(output_file, save_count)
while input_file.tell() < file_size:
# This should be pretty straightforward.
hunk_header_offset = input_file.tell()
hunk_id = persistence.read_uint16(input_file)
hunk_length = persistence.read_uint32(input_file)
hunk_payload_offset = input_file.tell()
actual_hunk_version = persistence.read_uint16(input_file)
expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id]
if expected_hunk_version != actual_hunk_version:
logger.error("convert_project_format_4_to_5: hunk %d version mismatch %d != %d", hunk_id, expected_hunk_version, actual_hunk_version)
return None
logger.debug("convert_project_format_4_to_5: file hunk %d", hunk_id)
if SAVEFILE_HUNK_LOADER != hunk_id:
input_file.seek(hunk_header_offset, os.SEEK_SET)
raw_hunk_length = (hunk_payload_offset - hunk_header_offset) + hunk_length
output_file.write(input_file.read(raw_hunk_length))
continue
# Write the as yet to be updated header.
persistence.write_uint16(output_file, hunk_id)
output_file_length_offset = output_file.tell()
persistence.write_uint32(output_file, 0)
output_file_payload_offset = output_file.tell()
persistence.write_uint16(output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_LOADER] + 1)
# Transform the hunk from input file to output file.
persistence.write_string(output_file, persistence.read_string(input_file)) # loader_system_name
data_size = persistence.read_uint32(input_file)
persistence.write_uint32(output_file, data_size) # loader_segments
output_file.write(input_file.read(data_size))
set_value = persistence.read_set_of_uint32s(input_file)
persistence.write_dict_uint32_to_list_of_uint32s(output_file, { k: [] for k in set_value }) # loader_relocated_addresses
set_value = persistence.read_set_of_uint32s(input_file)
persistence.write_set_of_uint32s(output_file, set_value) # loader_relocatable_addresses
persistence.write_uint16(output_file, persistence.read_uint16(input_file)) # loader_entrypoint_segment_id
persistence.write_int32(output_file, persistence.read_uint32(input_file)) # loader_entrypoint_offset
# Update the header length field, then fast forward to the end of the hunk.
new_hunk_length = output_file.tell() - output_file_payload_offset
output_file.seek(output_file_length_offset, os.SEEK_SET)
persistence.write_uint32(output_file, new_hunk_length)
output_file.seek(new_hunk_length, os.SEEK_CUR)
if output_file.tell() - output_file_payload_offset != new_hunk_length:
logger.error("convert_project_format_4_to_5: block length mismatch %d != %d", output_file.tell() - output_file_payload_offset, new_hunk_length)
return None
return output_file
def load_sourcedatainfo_hunk(f, program_data):
program_data.file_size = persistence.read_uint32(f)
program_data.file_checksum = persistence.read_bytes(f, 16)
def load_project(f, work_state=None):
logger.debug("file %s", f)
while True:
if work_state is not None and work_state.check_exit_update(0.1, "TEXT_LOAD_CONVERTING_PROJECT_FILE"):
return None
f.seek(0, os.SEEK_END)
file_size = f.tell()
f.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(f)
if savefile_id != SAVEFILE_ID:
logger.error("Save-file does not have first four bytes of '%X', has '%X' instead.", SAVEFILE_ID, savefile_id)
return None
savefile_version = persistence.read_uint16(f)
if savefile_version != SAVEFILE_VERSION:
new_f = None
if savefile_version == 2:
new_f = convert_project_format_2_to_3(f)
savefile_version = 3
elif savefile_version == 3:
new_f = convert_project_format_3_to_4(f)
savefile_version = 4
if new_f is None:
logger.error("load_project: save file is version %s, only version %s is supported at this time.", savefile_version, SAVEFILE_VERSION)
return None
f = new_f
logger.info("load_project: save file upgraded to version %d", savefile_version)
continue
break
logger.debug("bfile %s", f)
program_data = ProgramData()
program_data.save_count = persistence.read_uint32(f)
sourcedata_offset = sourcedata_length = None
while f.tell() < file_size:
if work_state is not None and work_state.check_exit_update(0.1 + 0.8 * (file_size-f.tell()), "TEXT_LOAD_READING_PROJECT_DATA"):
return None
hunk_id = persistence.read_uint16(f)
hunk_length = persistence.read_uint32(f)
expected_hunk_version = CURRENT_HUNK_VERSIONS[hunk_id]
offset0 = f.tell()
actual_hunk_version = persistence.read_uint16(f)
if SAVEFILE_HUNK_DISASSEMBLY == hunk_id:
load_disassembly_hunk(f, program_data)
elif SAVEFILE_HUNK_LOADER == hunk_id:
load_loader_hunk(f, program_data)
elif SAVEFILE_HUNK_LOADERINTERNAL == hunk_id:
load_loaderinternaldata_hunk(f, program_data)
elif SAVEFILE_HUNK_SOURCEDATAINFO == hunk_id:
load_sourcedatainfo_hunk(f, program_data)
elif SAVEFILE_HUNK_SOURCEDATA == hunk_id:
skip_bytes = (f.tell() - offset0)
sourcedata_offset, sourcedata_length = offset0 + skip_bytes, hunk_length - skip_bytes
f.seek(sourcedata_length, os.SEEK_CUR)
else:
logger.error("load_project encountered unknown hunk, with id: %d", hunk_id)
return None
offsetN = f.tell()
if offsetN - offset0 != hunk_length:
logger.error("load_project encountered hunk length mismatch, expected: %d, got: %d, hunk id: %d", hunk_length, offsetN - offset0, hunk_id)
return None
if work_state is not None and work_state.check_exit_update(0.95 * (file_size-f.tell()), "TEXT_LOAD_POSTPROCESSING"):
return None
if sourcedata_offset is not None:
logger.info("Caching input file segments from embedded source file.")
segments = program_data.loader_segments
for i in range(len(segments)):
loaderlib.cache_segment_data(f, segments, i, sourcedata_offset)
# Avoid doing relocations if there weren't any. e.g. binary files.
if len(program_data.loader_relocatable_addresses):
logger.info("Re-extracting relocations from embedded source file.")
file_info, data_types = loaderlib.load_file(f, None, file_offset=sourcedata_offset, file_length=sourcedata_length)
loaderlib.relocate_segment_data(segments, data_types, file_info.relocations_by_segment_id, program_data.loader_relocatable_addresses, program_data.loader_relocated_addresses)
program_data.input_file_cached = True
logger.info("Project loaded")
return program_data
def convert_project_format_3_to_4(input_file):
"""
This function should encapsulate all application-specific logic involved to
make it independent of as many changes as possible.
From version: 3.
To version: 4.
Modifications:
- disassembly hunk processor id has changed from string to uint32.
"""
SNAPSHOT_HUNK_VERSIONS = {
SAVEFILE_HUNK_SOURCEDATA: 1,
SAVEFILE_HUNK_SOURCEDATAINFO: 1,
SAVEFILE_HUNK_LOADER: 1,
SAVEFILE_HUNK_LOADERINTERNAL: 1,
SAVEFILE_HUNK_DISASSEMBLY: 1,
}
input_file.seek(0, os.SEEK_END)
file_size = input_file.tell()
input_file.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(input_file)
savefile_version = persistence.read_uint16(input_file)
if savefile_version != 3:
return None
logger.info("Upgrading save-file from version 3 to version 4: Processor id field..")
save_count = persistence.read_uint32(input_file)
output_file = tempfile.TemporaryFile()
persistence.write_uint32(output_file, savefile_id)
persistence.write_uint16(output_file, 4)
persistence.write_uint32(output_file, save_count)
while input_file.tell() < file_size:
# This should be pretty straightforward.
hunk_header_offset = input_file.tell()
hunk_id = persistence.read_uint16(input_file)
hunk_length = persistence.read_uint32(input_file)
hunk_payload_offset = input_file.tell()
actual_hunk_version = persistence.read_uint16(input_file)
expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id]
if expected_hunk_version != actual_hunk_version:
logger.error("convert_project_format_3_to_4: hunk %d version mismatch %d != %d", hunk_id, expected_hunk_version, actual_hunk_version)
return None
logger.debug("convert_project_format_3_to_4: file hunk %d", hunk_id)
# Copy unaffected hunks verbatim.
if hunk_id != SAVEFILE_HUNK_DISASSEMBLY:
input_file.seek(hunk_header_offset, os.SEEK_SET)
raw_hunk_length = (hunk_payload_offset - hunk_header_offset) + hunk_length
output_file.write(input_file.read(raw_hunk_length))
continue
## 1. Load the hunk payload.
branch_addresses = persistence.read_dict_uint32_to_set_of_uint32s(input_file)
reference_addresses = persistence.read_dict_uint32_to_set_of_uint32s(input_file)
symbols_by_address = persistence.read_dict_uint32_to_string(input_file)
post_segment_addresses = persistence.read_dict_uint32_to_list_of_uint32s(input_file)
flags = persistence.read_uint32(input_file)
processor_name = persistence.read_string(input_file)
# Reconstitute the segment block list.
num_blocks = persistence.read_uint32(input_file)
input_file_offset = input_file.tell()
block_data_length = hunk_length - (input_file_offset - hunk_payload_offset)
block_data_string = input_file.read(block_data_length)
#blocks = [ None ] * num_blocks
#for i in xrange(num_blocks):
# blocks[i] = read_SegmentBlock(input_file)
## 2. Write the generic hunk header, then the payload, then fill in the header.
# Only these two are likely to have been in use.
if processor_name == "m68k":
processor_id = loaderlib.constants.PROCESSOR_M680x0
elif processor_name == "mips":
processor_id = loaderlib.constants.PROCESSOR_MIPS
else:
logger.error("convert_project_format_3_to_4: unrecognised arch name %s", processor_name)
return None
logger.debug("convert_project_format_3_to_4: arch name %s maps to processor id %d", processor_name, processor_id)
# Write the as yet to be updated header.
persistence.write_uint16(output_file, hunk_id)
output_file_length_offset = output_file.tell()
persistence.write_uint32(output_file, 0)
output_file_payload_offset = output_file.tell()
persistence.write_uint16(output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_DISASSEMBLY] + 1)
# Write the payload in the modified format.
persistence.write_dict_uint32_to_set_of_uint32s(output_file, branch_addresses)
persistence.write_dict_uint32_to_set_of_uint32s(output_file, reference_addresses)
persistence.write_dict_uint32_to_string(output_file, symbols_by_address)
persistence.write_dict_uint32_to_list_of_uint32s(output_file, post_segment_addresses)
persistence.write_uint32(output_file, flags)
persistence.write_uint32(output_file, processor_id)
persistence.write_uint32(output_file, num_blocks)
output_file_offset = output_file.tell()
#for block in blocks:
# write_SegmentBlock(output_file, block)
output_file.write(block_data_string)
if output_file.tell() - output_file_offset != block_data_length:
logger.error("convert_project_format_3_to_4: block length mismatch %d != %d", output_file.tell() - output_file_offset, block_data_length)
return None
# Update the header length field, then fast forward to the end of the hunk.
new_hunk_length = output_file.tell() - output_file_payload_offset
output_file.seek(output_file_length_offset, os.SEEK_SET)
persistence.write_uint32(output_file, new_hunk_length)
output_file.seek(new_hunk_length, os.SEEK_CUR)
if output_file.tell() - output_file_payload_offset != new_hunk_length:
logger.error("convert_project_format_3_to_4: block length mismatch %d != %d", output_file.tell() - output_file_payload_offset, new_hunk_length)
return None
return output_file
def check_is_project_file(f):
f.seek(0, os.SEEK_SET)
return persistence.read_uint32(f) == SAVEFILE_ID
def check_is_project_file(f):
f.seek(0, os.SEEK_SET)
return persistence.read_uint32(f) == SAVEFILE_ID
def convert_project_format_3_to_4(input_file):
"""
This function should encapsulate all application-specific logic involved to
make it independent of as many changes as possible.
From version: 3.
To version: 4.
Modifications:
- disassembly hunk processor id has changed from string to uint32.
"""
SNAPSHOT_HUNK_VERSIONS = {
SAVEFILE_HUNK_SOURCEDATA: 1,
SAVEFILE_HUNK_SOURCEDATAINFO: 1,
SAVEFILE_HUNK_LOADER: 1,
SAVEFILE_HUNK_LOADERINTERNAL: 1,
SAVEFILE_HUNK_DISASSEMBLY: 1,
}
input_file.seek(0, os.SEEK_END)
file_size = input_file.tell()
input_file.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(input_file)
savefile_version = persistence.read_uint16(input_file)
if savefile_version != 3:
return None
logger.info(
"Upgrading save-file from version 3 to version 4: Processor id field.."
)
save_count = persistence.read_uint32(input_file)
output_file = tempfile.TemporaryFile()
persistence.write_uint32(output_file, savefile_id)
persistence.write_uint16(output_file, 4)
persistence.write_uint32(output_file, save_count)
while input_file.tell() < file_size:
# This should be pretty straightforward.
hunk_header_offset = input_file.tell()
hunk_id = persistence.read_uint16(input_file)
hunk_length = persistence.read_uint32(input_file)
hunk_payload_offset = input_file.tell()
actual_hunk_version = persistence.read_uint16(input_file)
expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id]
if expected_hunk_version != actual_hunk_version:
logger.error(
"convert_project_format_3_to_4: hunk %d version mismatch %d != %d",
hunk_id, expected_hunk_version, actual_hunk_version)
return None
logger.debug("convert_project_format_3_to_4: file hunk %d", hunk_id)
# Copy unaffected hunks verbatim.
if hunk_id != SAVEFILE_HUNK_DISASSEMBLY:
input_file.seek(hunk_header_offset, os.SEEK_SET)
raw_hunk_length = (hunk_payload_offset -
hunk_header_offset) + hunk_length
output_file.write(input_file.read(raw_hunk_length))
continue
## 1. Load the hunk payload.
branch_addresses = persistence.read_dict_uint32_to_set_of_uint32s(
input_file)
reference_addresses = persistence.read_dict_uint32_to_set_of_uint32s(
input_file)
symbols_by_address = persistence.read_dict_uint32_to_string(input_file)
post_segment_addresses = persistence.read_dict_uint32_to_list_of_uint32s(
input_file)
flags = persistence.read_uint32(input_file)
processor_name = persistence.read_string(input_file)
# Reconstitute the segment block list.
num_blocks = persistence.read_uint32(input_file)
input_file_offset = input_file.tell()
block_data_length = hunk_length - (input_file_offset -
hunk_payload_offset)
block_data_string = input_file.read(block_data_length)
#blocks = [ None ] * num_blocks
#for i in range(num_blocks):
# blocks[i] = read_SegmentBlock(input_file)
## 2. Write the generic hunk header, then the payload, then fill in the header.
# Only these two are likely to have been in use.
if processor_name == "m68k":
processor_id = loaderlib.constants.PROCESSOR_M680x0
elif processor_name == "mips":
processor_id = loaderlib.constants.PROCESSOR_MIPS
else:
logger.error(
"convert_project_format_3_to_4: unrecognised arch name %s",
processor_name)
return None
logger.debug(
"convert_project_format_3_to_4: arch name %s maps to processor id %d",
processor_name, processor_id)
# Write the as yet to be updated header.
persistence.write_uint16(output_file, hunk_id)
output_file_length_offset = output_file.tell()
persistence.write_uint32(output_file, 0)
output_file_payload_offset = output_file.tell()
persistence.write_uint16(
output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_DISASSEMBLY] + 1)
# Write the payload in the modified format.
persistence.write_dict_uint32_to_set_of_uint32s(
output_file, branch_addresses)
persistence.write_dict_uint32_to_set_of_uint32s(
output_file, reference_addresses)
persistence.write_dict_uint32_to_string(output_file,
symbols_by_address)
persistence.write_dict_uint32_to_list_of_uint32s(
output_file, post_segment_addresses)
persistence.write_uint32(output_file, flags)
persistence.write_uint32(output_file, processor_id)
persistence.write_uint32(output_file, num_blocks)
output_file_offset = output_file.tell()
#for block in blocks:
# write_SegmentBlock(output_file, block, program_data.dis_constant_pc_offset)
output_file.write(block_data_string)
if output_file.tell() - output_file_offset != block_data_length:
logger.error(
"convert_project_format_3_to_4: block length mismatch %d != %d",
output_file.tell() - output_file_offset, block_data_length)
return None
# Update the header length field, then fast forward to the end of the hunk.
new_hunk_length = output_file.tell() - output_file_payload_offset
output_file.seek(output_file_length_offset, os.SEEK_SET)
persistence.write_uint32(output_file, new_hunk_length)
output_file.seek(new_hunk_length, os.SEEK_CUR)
if output_file.tell() - output_file_payload_offset != new_hunk_length:
logger.error(
"convert_project_format_3_to_4: block length mismatch %d != %d",
output_file.tell() - output_file_payload_offset,
new_hunk_length)
return None
return output_file
def convert_project_format_4_to_5(input_file):
"""
This function should encapsulate all application-specific logic involved to
make it independent of as many changes as possible.
Version 4 -> 5.
Modifications:
- symbols by address has gone from a label to a structure with metadata.
"""
SNAPSHOT_HUNK_VERSIONS = {
SAVEFILE_HUNK_SOURCEDATA: 1,
SAVEFILE_HUNK_SOURCEDATAINFO: 1,
SAVEFILE_HUNK_LOADER: 1,
SAVEFILE_HUNK_LOADERINTERNAL: 1,
SAVEFILE_HUNK_DISASSEMBLY: 2,
}
input_file.seek(0, os.SEEK_END)
file_size = input_file.tell()
input_file.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(input_file)
savefile_version = persistence.read_uint16(input_file)
if savefile_version != 4:
return None
logger.info(
"Upgrading save-file from version 4 to version 5: symbols by address.."
)
save_count = persistence.read_uint32(input_file)
output_file = tempfile.TemporaryFile()
persistence.write_uint32(output_file, savefile_id)
persistence.write_uint16(output_file, 5)
persistence.write_uint32(output_file, save_count)
while input_file.tell() < file_size:
# This should be pretty straightforward.
hunk_header_offset = input_file.tell()
hunk_id = persistence.read_uint16(input_file)
hunk_length = persistence.read_uint32(input_file)
hunk_payload_offset = input_file.tell()
actual_hunk_version = persistence.read_uint16(input_file)
expected_hunk_version = SNAPSHOT_HUNK_VERSIONS[hunk_id]
if expected_hunk_version != actual_hunk_version:
logger.error(
"convert_project_format_4_to_5: hunk %d version mismatch %d != %d",
hunk_id, expected_hunk_version, actual_hunk_version)
return None
logger.debug("convert_project_format_4_to_5: file hunk %d", hunk_id)
if SAVEFILE_HUNK_LOADER != hunk_id:
input_file.seek(hunk_header_offset, os.SEEK_SET)
raw_hunk_length = (hunk_payload_offset -
hunk_header_offset) + hunk_length
output_file.write(input_file.read(raw_hunk_length))
continue
# Write the as yet to be updated header.
persistence.write_uint16(output_file, hunk_id)
output_file_length_offset = output_file.tell()
persistence.write_uint32(output_file, 0)
output_file_payload_offset = output_file.tell()
persistence.write_uint16(
output_file, SNAPSHOT_HUNK_VERSIONS[SAVEFILE_HUNK_LOADER] + 1)
# Transform the hunk from input file to output file.
persistence.write_string(
output_file,
persistence.read_string(input_file)) # loader_system_name
data_size = persistence.read_uint32(input_file)
persistence.write_uint32(output_file, data_size) # loader_segments
output_file.write(input_file.read(data_size))
set_value = persistence.read_set_of_uint32s(input_file)
persistence.write_dict_uint32_to_list_of_uint32s(
output_file, {k: []
for k in set_value}) # loader_relocated_addresses
set_value = persistence.read_set_of_uint32s(input_file)
persistence.write_set_of_uint32s(
output_file, set_value) # loader_relocatable_addresses
persistence.write_uint16(
output_file, persistence.read_uint16(
input_file)) # loader_entrypoint_segment_id
persistence.write_int32(
output_file,
persistence.read_uint32(input_file)) # loader_entrypoint_offset
# Update the header length field, then fast forward to the end of the hunk.
new_hunk_length = output_file.tell() - output_file_payload_offset
output_file.seek(output_file_length_offset, os.SEEK_SET)
persistence.write_uint32(output_file, new_hunk_length)
output_file.seek(new_hunk_length, os.SEEK_CUR)
if output_file.tell() - output_file_payload_offset != new_hunk_length:
logger.error(
"convert_project_format_4_to_5: block length mismatch %d != %d",
output_file.tell() - output_file_payload_offset,
new_hunk_length)
return None
return output_file
def load_sourcedatainfo_hunk(f, program_data):
program_data.file_size = persistence.read_uint32(f)
program_data.file_checksum = persistence.read_bytes(f, 16)
def load_project(f, work_state=None):
logger.debug("file %s", f)
while True:
if work_state is not None and work_state.check_exit_update(
0.1, "TEXT_LOAD_CONVERTING_PROJECT_FILE"):
return None
f.seek(0, os.SEEK_END)
file_size = f.tell()
f.seek(0, os.SEEK_SET)
savefile_id = persistence.read_uint32(f)
if savefile_id != SAVEFILE_ID:
logger.error(
"Save-file does not have first four bytes of '%X', has '%X' instead.",
SAVEFILE_ID, savefile_id)
return None
savefile_version = persistence.read_uint16(f)
if savefile_version != SAVEFILE_VERSION:
new_f = None
if savefile_version == 2:
new_f = convert_project_format_2_to_3(f)
savefile_version = 3
elif savefile_version == 3:
new_f = convert_project_format_3_to_4(f)
savefile_version = 4
elif savefile_version == 4:
new_f = convert_project_format_4_to_5(f)
savefile_version = 5
if new_f is None:
logger.error(
"load_project: save file is version %s, only version %s is supported at this time.",
savefile_version, SAVEFILE_VERSION)
return None
f = new_f
logger.info("load_project: save file upgraded to version %d",
savefile_version)
continue
break
logger.debug("bfile %s", f)
program_data = ProgramData()
program_data.save_count = persistence.read_uint32(f)
sourcedata_offset = sourcedata_length = None
while f.tell() < file_size:
if work_state is not None and work_state.check_exit_update(
0.1 + 0.8 *
(file_size - f.tell()), "TEXT_LOAD_READING_PROJECT_DATA"):
return None
hunk_id = persistence.read_uint16(f)
hunk_length = persistence.read_uint32(f)
expected_hunk_version = CURRENT_HUNK_VERSIONS[hunk_id]
offset0 = f.tell()
actual_hunk_version = persistence.read_uint16(f)
if SAVEFILE_HUNK_DISASSEMBLY == hunk_id:
load_disassembly_hunk(f, program_data)
elif SAVEFILE_HUNK_LOADER == hunk_id:
load_loader_hunk(f, program_data)
elif SAVEFILE_HUNK_LOADERINTERNAL == hunk_id:
load_loaderinternaldata_hunk(f, program_data)
elif SAVEFILE_HUNK_SOURCEDATAINFO == hunk_id:
load_sourcedatainfo_hunk(f, program_data)
elif SAVEFILE_HUNK_SOURCEDATA == hunk_id:
skip_bytes = (f.tell() - offset0)
sourcedata_offset, sourcedata_length = offset0 + skip_bytes, hunk_length - skip_bytes
f.seek(sourcedata_length, os.SEEK_CUR)
else:
logger.error("load_project encountered unknown hunk, with id: %d",
hunk_id)
return None
offsetN = f.tell()
if offsetN - offset0 != hunk_length:
logger.error(
"load_project encountered hunk length mismatch, expected: %d, got: %d, hunk id: %d",
hunk_length, offsetN - offset0, hunk_id)
return None
if work_state is not None and work_state.check_exit_update(
0.95 * (file_size - f.tell()), "TEXT_LOAD_POSTPROCESSING"):
return None
if sourcedata_offset is not None:
logger.info("Caching input file segments from embedded source file.")
segments = program_data.loader_segments
for i in range(len(segments)):
loaderlib.cache_segment_data(f, segments, i, sourcedata_offset)
# Avoid doing relocations if there weren't any. e.g. binary files.
if len(program_data.loader_relocatable_addresses):
logger.info("Re-extracting relocations from embedded source file.")
file_info, data_types = loaderlib.load_file(
f,
None,
file_offset=sourcedata_offset,
file_length=sourcedata_length)
loaderlib.relocate_segment_data(
segments, data_types, file_info.relocations_by_segment_id,
program_data.loader_relocatable_addresses,
program_data.loader_relocated_addresses)
program_data.input_file_cached = True
logger.info("Project loaded")
return program_data
