def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
super().parse(data_in, conf)
has_legacy_textures: bool = kwargs['has_legacy_textures']
end: int = kwargs['end']
rle = conf.game in (G.CROC_2_PS1, G.CROC_2_DEMO_PS1,
G.HARRY_POTTER_1_PS1, G.HARRY_POTTER_2_PS1)
textures: List[TextureData] = []
n_textures: int = int.from_bytes(data_in.read(4), 'little')
n_rows: int = int.from_bytes(data_in.read(4), 'little')
if n_textures > 4000 or 0 > n_rows > 4:
if conf.ignore_warnings:
warnings.warn(
f"Too much textures ({n_textures}, or incorrect row count {n_rows}."
f"It is most probably caused by an inaccuracy in my reverse engineering of the textures format."
)
else:
raise TexturesWarning(data_in.tell(), n_textures, n_rows)
# In Harry Potter, the last 16 textures are empty (full of 00 bytes)
n_stored_textures = n_textures - 16 if conf.game in (
G.HARRY_POTTER_1_PS1, G.HARRY_POTTER_2_PS1) else n_textures
for texture_id in range(n_stored_textures):
textures.append(TextureData.parse(data_in, conf))
if conf.game in (G.HARRY_POTTER_1_PS1, G.HARRY_POTTER_2_PS1):
data_in.seek(192, SEEK_CUR) # 16 textures x 12 bytes
n_idk_yet_1 = int.from_bytes(data_in.read(4), 'little')
n_idk_yet_2 = int.from_bytes(data_in.read(4), 'little')
data_in.seek(n_idk_yet_1 * cls.image_header_size, SEEK_CUR)
if has_legacy_textures: # Patch for legacy textures, see Textures documentation
data_in.seek(15360, SEEK_CUR)
if rle:
raw_textures = BytesIO(cls.image_bytes_size * b'\x00')
while data_in.tell() < end:
run = int.from_bytes(data_in.read(cls.rle_size),
'little',
signed=True)
if run < 0:
raw_textures.write(abs(run) * data_in.read(cls.rle_size))
elif run > 0:
raw_textures.write(data_in.read(cls.rle_size * run))
else:
raise ZeroRunLengthError(data_in.tell())
raw_textures.seek(0)
textures_data = raw_textures.read()
raw_textures.close()
if conf.game == G.CROC_2_DEMO_PS1: # Patch for Croc 2 Demo (non-dummy) last end offset error
data_in.seek(-2, SEEK_CUR)
else:
image_size = n_rows * (cls.image_bytes_size // 4)
padding_size = cls.image_bytes_size - image_size
textures_data = data_in.read(image_size) + padding_size * b'\x00'
legacy_alpha = conf.game in (G.CROC_2_DEMO_PS1,
G.CROC_2_DEMO_PS1_DUMMY)
return cls(n_rows, textures_data, legacy_alpha, textures)
def content_length(f: io.BufferedIOBase, preserve_pos=True):
if preserve_pos:
pos = f.tell()
f.seek(0, io.SEEK_END)
res = f.tell() // _VALUE_SIZE
if preserve_pos:
f.seek(pos)
return res
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
fallback_data = cls.fallback_parse_data(data_in)
size, start = super().parse(data_in, conf)
if conf.game == G.CROC_2_DEMO_PS1_DUMMY:
has_legacy_textures = False
titles = None
else:
tpsx_flags = int.from_bytes(data_in.read(4), 'little')
has_translated_titles = tpsx_flags & 16 != 0
has_legacy_textures = tpsx_flags & 8 != 0
has_title_and_demo_mode_data = tpsx_flags & 4 != 0
if has_title_and_demo_mode_data:
if has_translated_titles:
n_titles = int.from_bytes(data_in.read(4), 'little')
titles = [
data_in.read(48).strip(b'\0').decode('latin1')
for _ in range(n_titles)
]
else:
titles = [data_in.read(32).strip(b'\0').decode('latin1')]
data_in.seek(2052, SEEK_CUR)
else:
titles = None
texture_file = TextureFile.parse(
data_in,
conf,
has_legacy_textures=has_legacy_textures,
end=start + size)
cls.check_size(size, start, data_in.tell())
return cls(texture_file, titles, fallback_data)
def unpack(self, buffer: io.BufferedIOBase, bounding_box_has_unknown: bool = None):
data = self.STRUCT.unpack(buffer)
mesh_offset = buffer.tell()
bounding_box_offset = data.pop("__bounding_box_offset")
if bounding_box_offset != 0:
buffer.seek(bounding_box_offset)
self.bounding_box = BoundingBoxWithUnknown(buffer) if bounding_box_has_unknown else BoundingBox(buffer)
else:
self.bounding_box = None
buffer.seek(data.pop("__bone_offset"))
bone_count = data.pop("__bone_count")
self.bone_indices = list(unpack_from_buffer(buffer, f"<{bone_count}i",))
buffer.seek(data.pop("__face_set_offset"))
face_set_count = data.pop("__face_set_count")
self._face_set_indices = list(unpack_from_buffer(buffer, f"<{face_set_count}i"))
buffer.seek(data.pop("__vertex_buffer_offset"))
vertex_count = data.pop("__vertex_buffer_count")
self._vertex_buffer_indices = list(unpack_from_buffer(buffer, f"<{vertex_count}i"))
buffer.seek(mesh_offset)
self.set(**data)
def unpack(
self,
buffer: io.BufferedIOBase,
unicode: bool = None,
version: Version = None,
gx_lists: list[GXList] = None,
gx_list_indices: dict[int, int] = None,
):
if any(var is None
for var in (unicode, version, gx_lists, gx_list_indices)):
raise ValueError(
"Not all required keywords were passed to `Material.unpack()`."
)
data = self.STRUCT.unpack(buffer)
encoding = "utf-16-le" if unicode else "shift_jis_2004"
self.name = read_chars_from_buffer(buffer,
offset=data.pop("__name_offset"),
encoding=encoding)
self.mtd_path = read_chars_from_buffer(
buffer, offset=data.pop("__mtd_path_offset"), encoding=encoding)
gx_offset = data.pop("__gx_offset")
if gx_offset == 0:
self.gx_index = -1
elif gx_offset in gx_list_indices:
self.gx_index = gx_list_indices[gx_offset]
else:
gx_list_indices[gx_offset] = len(gx_lists)
material_offset = buffer.tell()
buffer.seek(gx_offset)
gx_lists.append(GXList(buffer, version))
buffer.seek(material_offset)
self.set(**data)
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args, **kwargs):
fallback_data = super().fallback_parse_data(data_in)
size, start = super().parse(data_in, conf)
idk1 = data_in.read(4)
n_idk_unique_textures = int.from_bytes(data_in.read(4), 'little')
if conf.game != G.CROC_2_DEMO_PS1_DUMMY:
data_in.seek(2048, SEEK_CUR)
else:
data_in.seek(2052, SEEK_CUR)
n_models_3d = int.from_bytes(data_in.read(4), 'little')
models_3d = [Model3DData.parse(data_in, conf) for _ in range(n_models_3d)]
n_animations = int.from_bytes(data_in.read(4), 'little')
animations = [AnimationData.parse(data_in, conf) for _ in range(n_animations)]
if conf.game in (G.CROC_2_PS1, G.CROC_2_DEMO_PS1):
n_dpsx_legacy_textures = int.from_bytes(data_in.read(4), 'little')
data_in.seek(n_dpsx_legacy_textures * 3072, SEEK_CUR)
n_scripts = int.from_bytes(data_in.read(4), 'little')
scripts = [ScriptData.parse(data_in, conf) for _ in range(n_scripts)]
level_file = LevelFile.parse(data_in, conf)
# FIXME End of Croc 2 & Croc 2 Demo Dummies' level files aren't reversed yet
if conf.game not in (G.CROC_2_PS1, G.CROC_2_DEMO_PS1_DUMMY):
cls.check_size(size, start, data_in.tell())
return cls(models_3d, animations, scripts, level_file, fallback_data)
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
super().parse(data_in, conf)
data_in.seek(72, SEEK_CUR)
n_vertices = int.from_bytes(data_in.read(4), 'little')
data_in.seek(8, SEEK_CUR)
n_faces = int.from_bytes(data_in.read(4), 'little')
if n_vertices > 1000 or n_faces > 1000:
if conf.ignore_warnings:
warnings.warn(
f"Too much vertices or faces ({n_vertices} vertices, {n_faces} faces)."
f"It is most probably caused by an inaccuracy in my reverse engineering of the models format."
)
else:
raise Models3DWarning(data_in.tell(), n_vertices, n_faces)
data_in.seek(4, SEEK_CUR)
n_bounding_box_info = int.from_bytes(
data_in.read(2), 'little') + int.from_bytes(
data_in.read(2), 'little') + int.from_bytes(
data_in.read(2), 'little')
if conf.game in (G.CROC_2_PS1, G.CROC_2_DEMO_PS1,
G.CROC_2_DEMO_PS1_DUMMY):
data_in.seek(2, SEEK_CUR)
elif conf.game in (G.HARRY_POTTER_1_PS1, G.HARRY_POTTER_2_PS1):
data_in.seek(6, SEEK_CUR)
return cls(n_vertices, n_faces, n_bounding_box_info)
def writeto(self, buffer: BufferedIOBase):
self.first_node_offset = 0x20
# do strings
string_pool_base_offset = self.first_node_offset + len(self.nodes) * 12
buffer.seek(string_pool_base_offset)
for node in self.nodes:
node.string_offset = buffer.tell() - string_pool_base_offset
buffer.write(node.name.encode('ASCII'))
buffer.write(b'\x00')
self.all_node_size = buffer.tell() - self.first_node_offset
# padding before data section to 16
pad = (32 - (buffer.tell() % 32))
if pad == 32:
pad = 0
buffer.write(b'\x00' * pad)
self.data_offset = buffer.tell()
buffer.seek(0)
buffer.write(MAGIC_HEADER)
buffer.write(struct.pack('>I', self.first_node_offset))
buffer.write(struct.pack('>I', self.all_node_size))
buffer.write(struct.pack('>I', self.data_offset))
buffer.seek(self.first_node_offset)
cur_data_offset = self.data_offset
for i, node in enumerate(self.nodes):
if node.node_type == b'\x00':
# todo modified/unmodified
node.new_data_offset = cur_data_offset
buffer.seek(cur_data_offset)
node.write_data_to(self, buffer)
cur_data_offset += node.get_length()
# pad to 32
pad = (32 - (buffer.tell() % 32))
if pad == 32:
pad = 0
cur_data_offset += pad
buffer.seek(self.first_node_offset + i * 12)
node.write_header_to(buffer)
buffer.seek(cur_data_offset)
# final padding to 16
pad = (16 - (cur_data_offset % 16))
if pad == 16:
pad = 0
buffer.write(b'\x00' * pad)
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
size, start = super().parse(data_in, conf)
spsx_section: SPSXSection = kwargs['spsx_section']
if size != 0:
if SPSXFlags.HAS_LEVEL_SFX in spsx_section.spsx_flags:
spsx_section.level_sfx_groups.parse_vags(data_in, conf)
spsx_section.level_sfx_mapping.parse_mapping(
spsx_section.level_sfx_groups)
data_in.seek(2048 * math.ceil(data_in.tell() / 2048))
spsx_section.dialogues_bgms.parse_vags(data_in, conf)
if conf.game == G.HARRY_POTTER_2_PS1:
data_in.seek(2048 * math.ceil(data_in.tell() / 2048))
cls.check_size(size, start, data_in.tell())
return cls(spsx_section)
def __fill_buf(self, f_in: BufferedIOBase) -> bool:
next_bytes = f_in.read(self.read_len)
if len(next_bytes) == 0:
return False
cur_process = f_in.tell() * 100 // self.length
if cur_process != self.process:
self.process = cur_process
print(str(cur_process) + "%")
self.buf.extend(next_bytes)
return True
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
super().parse(data_in, conf)
n_flags: int = int.from_bytes(data_in.read(4), 'little')
data_in.seek(4, SEEK_CUR)
n_total_frames: int = int.from_bytes(data_in.read(4), 'little')
has_additional_frame_data_value = int.from_bytes(
data_in.read(4), 'little')
has_additional_data: bool = has_additional_frame_data_value == 0
n_stored_frames = 0
if conf.game in (G.CROC_2_PS1, G.CROC_2_DEMO_PS1,
G.CROC_2_DEMO_PS1_DUMMY):
n_inter_frames = int.from_bytes(data_in.read(4), 'little')
if n_inter_frames != 0:
n_stored_frames = n_total_frames
data_in.seek(4, SEEK_CUR)
else: # Harry Potter 1 & 2
data_in.seek(8, SEEK_CUR)
n_inter_frames = 0
n_vertex_groups: int = int.from_bytes(data_in.read(4), 'little')
data_in.seek(4, SEEK_CUR)
if conf.game in (G.HARRY_POTTER_1_PS1, G.HARRY_POTTER_2_PS1):
n_stored_frames = int.from_bytes(data_in.read(4), 'little')
data_in.seek(12, SEEK_CUR)
flags = [data_in.read(4) for _ in range(n_flags)]
if has_additional_data:
data_in.seek(8 * n_total_frames, SEEK_CUR)
data_in.seek(4 * n_total_frames, SEEK_CUR) # Total frames info
data_in.seek(n_inter_frames * cls.inter_frames_header_size,
SEEK_CUR) # Inter-frames header
if conf.game in (G.HARRY_POTTER_1_PS1,
G.HARRY_POTTER_2_PS1) or n_inter_frames != 0:
data_in.seek(4 * n_stored_frames, SEEK_CUR) # Stored frames info
if n_stored_frames == 0 or n_inter_frames != 0: # Rotation matrices
old_animation_format = True
n_stored_frames = n_total_frames
else: # Unit quaternions
old_animation_format = False
if n_total_frames > 500 or n_total_frames == 0:
if conf.ignore_warnings:
warnings.warn(
f"Too much frames in animation (or no frame): {n_total_frames} frames."
f"It is most probably caused by an inaccuracy in my reverse engineering of the textures format."
)
else:
raise AnimationsWarning(data_in.tell(), n_total_frames)
return cls(n_total_frames, n_stored_frames, n_vertex_groups, n_flags,
has_additional_data, flags, old_animation_format,
n_inter_frames)
def unpack(self, buffer: io.BufferedIOBase):
data = self.STRUCT.unpack(buffer)
layout_offset = buffer.tell()
buffer.seek(data.pop("__member_offset"))
struct_offset = 0
self.members = []
for _ in range(data.pop("__member_count")):
member = LayoutMember(buffer, dict(struct_offset=struct_offset))
self.members.append(member)
struct_offset += member.layout_type.size()
buffer.seek(layout_offset)
def read_file_to_socket(file: BufferedIOBase, sock: socket, size):
"""
Copies an entire file to a socket, logging progress if file is big.
"""
logProgress = False
if size > big_file_threshold:
logProgress = True
try:
while file.tell() < size:
bytes = file.read(buffer_size)
sock.sendall(bytes)
if logProgress:
print("Sending file",
file.tell(),
"/",
size,
"bytes sent",
end='\r')
if logProgress: print()
except Exception:
print("Error while sending file. Maybe the other side refused?")
def extract(meta: io.BufferedIOBase, binFile: io.BufferedIOBase,
rmdp: io.BufferedIOBase, filter, target):
pc_bin = 0
fmt = '>'
array = binFile.read(1)
if array[0] == 0:
pc_bin = 1
fmt = '<'
file_version, num_dirs, num_files = struct.unpack(fmt + 'iii',
binFile.read(4 * 3))
binFile.seek(0x9D)
dirs = []
full_names = []
for i in range(num_dirs):
_, parent, _, dirname, _, _, _ = _read(binFile, fmt + 'qqiqiqq')
dirs.append((parent, dirname))
r = []
for i in range(num_files):
_, dir_index, _, filename_offset, content_offset, content_length = _read(
binFile, fmt + 'qqiqqq')
_skip(binFile, 16)
r.append((filename_offset, content_offset, content_length, dir_index))
_skip(binFile, 44)
data_start = binFile.tell()
for parent, dirname_offset in dirs:
if dirname_offset == -1:
full_names.append(target)
else:
dirname = read_text(binFile, data_start + dirname_offset)
parent = full_names[parent]
full_names.append(os.path.join(parent, dirname))
for i, (filename_offset, content_offset, content_length,
dir_index) in enumerate(r):
filename = read_text(binFile, data_start + filename_offset)
print(i, '/', len(r), end='\r')
dirname = full_names[dir_index]
os.makedirs(dirname, exist_ok=True)
full_name = os.path.join(dirname, filename)
with open(full_name, 'wb') as outf:
outf.write(copy_data(rmdp, content_offset, content_length))
def uncompress_num(f_in: BufferedIOBase) -> int:
pos = f_in.tell()
compressed_num = f_in.read(10)
if len(compressed_num) == 0:
return None
num = 0
for i in range(len(compressed_num)):
low = compressed_num[i]
if low < 128:
num += low << (i * 7)
else:
num += (low - 128) << (i * 7)
f_in.seek(pos + i + 1)
return num
else:
raise RuntimeError("compressed_num is cut, should enlarge it")
def copy_byte_range(
infile: io.BufferedIOBase,
outfile: io.BufferedIOBase,
start: int = None,
stop: int = None,
bufsize: int = 16 * 1024,
):
"""Like shutil.copyfileobj, but only copy a range of the streams.
Both start and stop are inclusive.
"""
if start is not None:
infile.seek(start)
while True:
to_read = min(bufsize, stop + 1 - infile.tell() if stop else bufsize)
buf = infile.read(to_read)
if not buf:
break
outfile.write(buf)
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
fallback_data = cls.fallback_parse_data(data_in)
size, start = super().parse(data_in, conf)
n_zones = int.from_bytes(data_in.read(4), 'little')
n_idk1 = int.from_bytes(data_in.read(4), 'little')
idk1 = [data_in.read(32) for _ in range(n_idk1)]
n_chunks_per_zone = []
for _ in range(n_zones):
data_in.seek(2, SEEK_CUR)
n_chunks_per_zone.append(data_in.read(1)[0])
data_in.seek(9, SEEK_CUR)
chunks_zones = []
for n_chunks in n_chunks_per_zone:
chunks_zones.append([
int.from_bytes(data_in.read(2), 'little')
for _ in range(n_chunks)
])
cls.check_size(size, start, data_in.tell())
return cls(idk1, chunks_zones, fallback_data)
def _filesize(f: io.BufferedIOBase):
pos = f.tell()
f.seek(0, io.SEEK_END)
result = f.tell()
f.seek(pos)
return result
def _is_dcx(buffer: io.BufferedIOBase):
offset = buffer.tell()
is_dcx = buffer.read(4) == b"DCX\0"
buffer.seek(offset)
return is_dcx
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
size, start = super().parse(data_in, conf)
spsx_flags: SPSXFlags = SPSXFlags(
int.from_bytes(data_in.read(4), 'little'))
assert spsx_flags & 0x2 == 0 # Bit 1 is always unset
# Bit 0 and 4 are identical
assert (SPSXFlags.HAS_AMBIENT_TRACKS
in spsx_flags) == (SPSXFlags.HAS_AMBIENT_TRACKS_ in spsx_flags)
logging.debug(f"Flags: {str(spsx_flags)}")
n_sfx = int.from_bytes(data_in.read(4), 'little')
logging.debug(f"sound effects count: {n_sfx}")
common_sfx: Optional[CommonSFXContainer] = None
if SPSXFlags.HAS_COMMON_SFX_AND_DIALOGUES_BGMS in spsx_flags:
common_sfx = CommonSFXContainer(
EffectSound.parse(data_in, conf) for _ in range(n_sfx))
ambient_tracks: Optional[AmbientContainer] = None
if SPSXFlags.HAS_AMBIENT_TRACKS in spsx_flags:
ambient_tracks_headers_size = int.from_bytes(
data_in.read(4), 'little')
assert ambient_tracks_headers_size % 20 == 0
n_ambient_tracks = ambient_tracks_headers_size // 20
ambient_tracks = AmbientContainer(
AmbientSound.parse(data_in, conf)
for _ in range(n_ambient_tracks))
# Level sound effects groups
idk1 = None
idk2 = None
level_sfx_groups: Optional[LevelSFXContainer] = None
level_sfx_mapping = None
if SPSXFlags.HAS_LEVEL_SFX in spsx_flags:
n_level_sfx_groups = int.from_bytes(data_in.read(4), 'little')
assert n_level_sfx_groups < 16 #
idk1 = int.from_bytes(data_in.read(4), 'little')
idk2 = int.from_bytes(data_in.read(4), 'little')
n_unique_level_sfx = int.from_bytes(data_in.read(4), 'little')
level_sfx_groups = LevelSFXContainer(
LevelSFXGroupContainer.parse(data_in, conf)
for _ in range(n_level_sfx_groups))
level_sfx_groups.parse_groups(data_in, conf)
assert n_unique_level_sfx <= level_sfx_groups.n_sounds
level_sfx_mapping = LevelSFXMapping.parse(
data_in, conf, n_unique_level_sfx=n_unique_level_sfx)
# Dialogues & BGMs
dialogues_bgms: Optional[DialoguesBGMsContainer] = None
n_dialogues_bgms = int.from_bytes(data_in.read(4), 'little')
if SPSXFlags.HAS_COMMON_SFX_AND_DIALOGUES_BGMS in spsx_flags:
# Gap between level sound effects and dialogues/BGMs
end_gap = int.from_bytes(data_in.read(4), 'little')
assert (end_gap != 0) == (SPSXFlags.HAS_LEVEL_SFX in spsx_flags)
logging.debug(f"END (DNE) gap: {end_gap}")
dialogues_bgms = DialoguesBGMsContainer(
DialogueBGMSound.parse(data_in, conf)
for _ in range(n_dialogues_bgms))
# Common sound effects audio data
common_sfx_total_size = int.from_bytes(data_in.read(4), 'little')
assert common_sfx_total_size == common_sfx.size
common_sfx.parse_vags(data_in, conf)
# Ambient tracks audio data
if SPSXFlags.HAS_AMBIENT_TRACKS in spsx_flags:
ambient_tracks_total_size = int.from_bytes(data_in.read(4),
'little')
assert ambient_tracks_total_size == ambient_tracks.size
ambient_tracks.parse_vags(data_in, conf)
cls.check_size(size, start, data_in.tell())
return cls(spsx_flags, common_sfx, ambient_tracks, level_sfx_groups,
level_sfx_mapping, idk1, idk2, dialogues_bgms)
def read_chars_from_buffer(
buffer: io.BufferedIOBase,
offset: int = None,
length: int = None,
reset_old_offset=True,
encoding: str = None,
strip=True,
) -> tp.Union[str, bytes]:
"""Read characters from a buffer (type IOBase). Use 'read_chars_from_bytes' if your data is already in bytes format.
Args:
buffer (io.BufferedIOBase): byte-format data stream to read from.
offset: offset to `seek()` in buffer before starting to read characters. Defaults to current offset (None).
reset_old_offset: if True, and 'offset' is not None, the buffer offset will be restored to its original position
(at function call time) before returning. (Default: True)
length: number of characters to read (i.e. the length of the returned string). If None (default), characters
will be read until a null termination is encountered. Otherwise, if a length is specified, any spaces at
the end of the string will be stripped, then any nulls at the end will be stripped.
encoding: attempt to decode characters in this encoding before returning. If 'utf-16-*' is specified, this
function will infer that characters are two bytes long (and null terminations will be two bytes). Otherwise,
it assumes they are one byte long. You can decode the characters yourself if you want to use another
multiple-bytes-per-character encoding.
strip: remove trailing spaces and nulls (default: True).
"""
if length == 0:
if not reset_old_offset and not isinstance(buffer, bytes):
buffer.seek(offset)
return "" if encoding is not None else b""
if isinstance(buffer, bytes):
buffer = io.BytesIO(buffer)
chars = []
old_offset = None
bytes_per_char = 2 if encoding is not None and encoding.replace(
"-", "").startswith("utf16le") else 1
if offset is not None:
old_offset = buffer.tell()
buffer.seek(offset)
while 1:
c = buffer.read(bytes_per_char)
if not c and length is None:
raise ValueError(
"Ran out of bytes to read before null termination was found.")
if length is None and c == b"\x00" * bytes_per_char:
# Null termination.
array = b"".join(chars)
if reset_old_offset and old_offset is not None:
buffer.seek(old_offset)
if encoding is not None:
return array.decode(encoding)
return array
else:
chars.append(c)
if len(chars) == length:
if reset_old_offset and old_offset is not None:
buffer.seek(old_offset)
stripped_array = b"".join(
chars) # used to strip spaces as well, but not anymore
if strip:
stripped_array.rstrip() # remove spaces
while stripped_array.endswith(b"\0\0" if bytes_per_char ==
2 else b"\0"):
stripped_array = stripped_array[:
-bytes_per_char] # remove (pairs of) null characters
if encoding is not None:
return stripped_array.decode(encoding)
return stripped_array
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
super().parse(data_in, conf)
n_chunk_models = int.from_bytes(data_in.read(4), 'little')
_chunk_model_headers = [
Model3DHeader.parse(data_in, conf) for _ in range(n_chunk_models)
]
chunk_models = [
LevelGeom3DData.parse(data_in,
conf,
header=_chunk_model_headers[i])
for i in range(n_chunk_models)
]
if conf.game != G.CROC_2_DEMO_PS1_DUMMY:
data_in.seek(8, SEEK_CUR)
n_sub_chunks = int.from_bytes(data_in.read(4), 'little')
n_idk1 = int.from_bytes(data_in.read(4), 'little')
data_in.seek(4 * n_idk1, SEEK_CUR)
assert n_sub_chunks == int.from_bytes(data_in.read(4), 'little')
n_actors_instances = int.from_bytes(data_in.read(2), 'little')
data_in.seek(6 if conf.game != G.CROC_2_DEMO_PS1_DUMMY else 2,
SEEK_CUR)
n_total_chunks = int.from_bytes(data_in.read(4), 'little')
n_chunk_columns = int.from_bytes(data_in.read(4), 'little')
n_chunk_rows = int.from_bytes(data_in.read(4), 'little')
if conf.game != G.CROC_2_DEMO_PS1_DUMMY:
n_lighting_headers = int.from_bytes(data_in.read(2), 'little')
n_add_sub_chunks_lighting = int.from_bytes(data_in.read(2),
'little')
idk3 = int.from_bytes(data_in.read(4), 'little')
else:
n_lighting_headers, n_add_sub_chunks_lighting = None, None
n_idk4 = int.from_bytes(data_in.read(4), 'little')
data_in.seek(116 if conf.game != G.CROC_2_DEMO_PS1_DUMMY else 80,
SEEK_CUR)
_chunks_matrix: List[Union[List[int], Optional[int]]] = \
[int.from_bytes(data_in.read(4), 'little') for _ in range(n_total_chunks)]
_sub_chunks_height = {}
chunks_info_start_offset = data_in.tell()
def parse_chunks_info(offset: int, chunks_ids_list: List[int]):
data_in.seek(chunks_info_start_offset + offset)
chunks_ids_list.append(int.from_bytes(data_in.read(4), 'little'))
linked_chunk_offset = int.from_bytes(data_in.read(4), 'little')
if linked_chunk_offset != 0xFFFFFFFF:
return parse_chunks_info(linked_chunk_offset, chunks_ids_list)
else:
return chunks_ids_list
for i in range(n_chunk_rows):
for j in range(n_chunk_columns):
index = i * n_chunk_columns + j
chunk_info_offset = _chunks_matrix[index]
if chunk_info_offset != 0xFFFFFFFF:
sub_chunk_ids = parse_chunks_info(chunk_info_offset, [])
_chunks_matrix[index] = sub_chunk_ids
for sub_chunk_id in sub_chunk_ids:
_sub_chunks_height[sub_chunk_id] = (i, j)
else:
_chunks_matrix[index] = None
data_in.seek(chunks_info_start_offset + 8 * n_sub_chunks)
if conf.game != G.CROC_2_DEMO_PS1_DUMMY:
header256bytes = data_in.read(256)
n_zone_ids = int.from_bytes(data_in.read(4), 'little')
zone_ids = [
int.from_bytes(data_in.read(4), 'little')
for _ in range(n_zone_ids)
]
assert n_zone_ids == n_total_chunks
if data_in.read(4) == b'fvw\x00':
fvw_data = [data_in.read(2) for _ in range(n_total_chunks)]
else:
fvw_data = None
data_in.seek(-4, SEEK_CUR)
else:
zone_ids = None
fvw_data = None
_sub_chunks_rotation = {}
for i in range(n_sub_chunks):
rotation = int.from_bytes(data_in.read(4), 'big')
assert rotation in (0, 4, 8, 12)
_sub_chunks_rotation[i] = ChunkRotation(rotation)
assert data_in.read(4) == b'\x00\x00\x00\x00'
x = int.from_bytes(data_in.read(4), 'little')
y = int.from_bytes(data_in.read(4), 'little')
z = int.from_bytes(data_in.read(4), 'little')
assert data_in.read(4) == b'\x00\x00\x00\x00'
assert x == 2048 + 4096 * _sub_chunks_height[i][
1] # Chunks are 4096-large, so +2048 for the chunk's center
assert z == 2048 + 4096 * _sub_chunks_height[i][0]
_sub_chunks_height[i] = y
chunks_models_mapping = [
int.from_bytes(data_in.read(4), 'little')
for _ in range(n_sub_chunks)
]
if conf.game != G.CROC_2_DEMO_PS1_DUMMY:
lighting_headers = [
data_in.read(84) for _ in range(n_lighting_headers)
]
idk_4 = [data_in.read(36) for _ in range(n_idk4)]
for i in range(n_actors_instances):
data_in.seek(24, SEEK_CUR)
actor_offset = int.from_bytes(data_in.read(4), 'little')
data_in.seek(32, SEEK_CUR)
actor_sound_level = int.from_bytes(data_in.read(4), 'little')
if conf.game not in (G.CROC_2_DEMO_PS1, G.CROC_2_DEMO_PS1_DUMMY):
add_models_mapping = []
for i in range(n_add_sub_chunks_lighting):
data_in.seek(16, SEEK_CUR)
add_models_mapping.append(
int.from_bytes(data_in.read(4), 'little'))
data_in.seek(4, SEEK_CUR)
n_idk2 = int.from_bytes(data_in.read(4), 'little')
data_in.seek(32 * n_idk2, SEEK_CUR) # TODO Reverse this
else:
add_models_mapping = None
data_in.seek(32 * n_sub_chunks,
SEEK_CUR) # Two different 32-bytes long structures
data_in.seek(32 * n_sub_chunks, SEEK_CUR)
data_in.seek(32 if conf.game == G.CROC_2_DEMO_PS1 else 92,
SEEK_CUR)
if conf.game == G.CROC_2_PS1:
data_in.seek(30732, SEEK_CUR)
elif conf.game != G.CROC_2_DEMO_PS1_DUMMY and n_sub_chunks != 0:
sub_chunks_n_lighting = [
int.from_bytes(data_in.read(4), 'little')
for _ in range(n_sub_chunks)
]
sub_chunks_n_add_lighting = [
int.from_bytes(data_in.read(4), 'little')
for _ in range(n_add_sub_chunks_lighting)
]
for model_id in range(n_sub_chunks):
for i in range(sub_chunks_n_lighting[model_id]):
size = 4 * chunk_models[
chunks_models_mapping[model_id]].n_vertices
data_in.seek(size, SEEK_CUR)
for model_id in range(n_add_sub_chunks_lighting):
for i in range(sub_chunks_n_add_lighting[model_id]):
size = 4 * chunk_models[
add_models_mapping[model_id]].n_vertices
data_in.seek(size, SEEK_CUR)
if conf.game != G.CROC_2_DEMO_PS1: # Not present in Croc 2 Demo Dummy
idk_size = int.from_bytes(data_in.read(4), 'little')
if idk_size != 0:
data_in.seek(4 + idk_size, SEEK_CUR)
else:
data_in.seek(-4, SEEK_CUR)
n_idk3 = int.from_bytes(data_in.read(4), 'little')
if n_idk3 == 0:
data_in.seek(-4, SEEK_CUR)
idk3 = [
int.from_bytes(data_in.read(40), 'little')
for _ in range(n_idk3)
]
data_in.seek(12, SEEK_CUR)
chunks_holders = []
for i in range(n_total_chunks):
if _chunks_matrix[i] is not None:
sub_chunks = [
SubChunk(chunk_models[chunks_models_mapping[sub_chunk_id]],
_sub_chunks_height[sub_chunk_id],
_sub_chunks_rotation[sub_chunk_id])
for sub_chunk_id in _chunks_matrix[i]
]
if conf.game != G.CROC_2_DEMO_PS1_DUMMY:
chunks_holders.append(
ChunkHolder(sub_chunks, zone_ids[i],
fvw_data[i] if fvw_data else None))
else:
chunks_holders.append(ChunkHolder(sub_chunks))
else:
chunks_holders.append(
ChunkHolder(zone_id=zone_ids[i],
fvw_data=fvw_data[i] if fvw_data else None)
if conf.game != G.CROC_2_DEMO_PS1_DUMMY else ChunkHolder())
return cls(
ChunksMatrix(chunks_holders, chunk_models, n_chunk_rows,
n_chunk_columns, zone_ids is not None))
def serialize_vags(self, data_out: BufferedIOBase, conf: Configuration):
for group in self:
data_out.seek(2048 * math.ceil(data_out.tell() / 2048))
group.serialize_vags(data_out, conf)
def parse_vags(self, data_in: BufferedIOBase, conf: Configuration):
for sound in self:
data_in.seek(2048 * math.ceil(data_in.tell() / 2048))
sound.parse_vag(data_in, conf)
def __init__(self, stream: io.BufferedIOBase, new: Version = None):
"""Parse a MIX from `stream`, which must be a buffered file object.
If `new` is not None, initialize an empty MIX of this version instead.
MixParseError is raised on parsing errors.
"""
# Initialize mandatory attributes
self._dirty = False
self._stream = None
self._open = []
# If stream is, for example, a raw I/O object, files could be destroyed
# without ever raising an error, so check this.
if not isinstance(stream, io.BufferedIOBase):
raise TypeError("`stream` must be an instance of io.BufferedIOBase")
if not stream.readable():
raise ValueError("`stream` must be readable")
if not stream.seekable():
raise ValueError("`stream` must be seekable")
if new is not None:
# Start empty (new file)
if type(new) is not Version:
raise TypeError("`new` must be a Version enumeration member or None")
if version is Version.RG:
raise NotImplementedError("RG MIX files are not yet supported")
self._stream = stream
self._index = {}
self._contents = []
self._version = version
self._flags = 0
return
# Parse an existing file
filesize = stream.seek(0, io.SEEK_END)
if filesize <= 6:
raise MixParseError("File too small")
stream.seek(0)
first4 = stream.read(4)
if first4 == b"MIX1":
raise NotImplementedError("RG MIX files are not yet supported")
elif first4[:2] == b"\x00\x00":
# It seems we have a RA or TS MIX so check the flags
flags = int.from_bytes(first4[2:], "little")
if flags > 3:
raise MixParseError("Unsupported properties")
if flags & 2:
raise NotImplementedError("Encrypted MIX files are not yet supported")
# FIXME HERE: 80 bytes of westwood key_source if encrypted,
# to create a 56 byte long blowfish key from it.
#
# They are followed by a number of 8 byte blocks,
# the first of them decrypting to filecount and bodysize.
# Encrypted TS MIXes have a key.ini we can check for later,
# so at this point assume Version.TS only if unencrypted.
# Stock RA MIXes seem to be always encrypted.
version = Version.TS
# RA/TS MIXes hold their filecount after the flags,
# whilst for TD MIXes their first two bytes are the filecount.
filecount = int.from_bytes(stream.read(2), "little")
else:
version = Version.TD
flags = 0
filecount = int.from_bytes(first4[:2], "little")
stream.seek(2)
# From here it's the same for every unencrypted MIX
bodysize = int.from_bytes(stream.read(4), "little")
indexoffset = stream.tell()
indexsize = filecount * 12
bodyoffset = indexoffset + indexsize
# Check if data is sane
# FIXME: Checksummed MIXes have 20 additional bytes after the body.
if filesize - bodyoffset != bodysize:
raise MixParseError("Incorrect filesize or invalid header")
# OK, time to read the index
index = {}
for key, offset, size in struct.iter_unpack("<LLL", stream.read(indexsize)):
offset += bodyoffset
if offset + size > filesize:
raise MixParseError("Content extends beyond end of file")
index[key] = _MixNode(key, offset, size, size, None)
if len(index) != filecount:
raise MixParseError("Duplicate key")
# Now read the names
# TD/RA: 1422054725; TS: 913179935
for dbkey in (1422054725, 913179935):
if dbkey in index:
stream.seek(index[dbkey].offset)
header = stream.read(32)
if header != b"XCC by Olaf van der Spek\x1a\x04\x17'\x10\x19\x80\x00":
continue
dbsize = int.from_bytes(stream.read(4), "little") # Total filesize
if dbsize != index[dbkey].size or dbsize > 16777216:
raise MixParseError("Invalid name table")
# Skip four bytes for XCC type; 0 for LMD, 2 for XIF
# Skip four bytes for DB version; Always zero
stream.seek(8, io.SEEK_CUR)
gameid = int.from_bytes(stream.read(4), "little") # XCC Game ID
# XCC saves alias numbers, so converting them
# to `Version` is not straight forward.
# FIXME: Check if Dune games and Nox also use MIX files
if gameid == 0:
if version is not Version.TD:
continue
elif gameid == 1:
version = Version.RA
elif 2 <= gameid <= 6 or gameid == 15:
version = Version.TS
else:
continue
namecount = int.from_bytes(stream.read(4), "little")
bodysize = dbsize - 53 # Size - Header - Last byte
namelist = stream.read(bodysize).split(b"\x00") if bodysize else []
if len(namelist) != namecount:
raise MixParseError("Invalid name table")
# Remove Database from index
del index[dbkey]
# Add names to index
names = False
for name in namelist:
name = name.decode(ENCODING, "ignore")
key = genkey(name, version)
if key in index:
index[key].name = name
names = True
# XCC sometimes puts two Databases in a file by mistake,
# so if no names were found, give it another try
if names: break
# Create a sorted list of all contents
contents = sorted(index.values(), key=lambda node: node.offset)
# Calculate alloc values
# This is the size up to wich a file may grow without needing a move
for i in range(len(contents) - 1):
contents[i].alloc = contents[i+1].offset - contents[i].offset
if contents[i].alloc < contents[i].size:
raise MixParseError("Overlapping file boundaries")
# Populate the object
self._stream = stream
self._version = version
self._index = index
self._contents = contents
self._flags = flags
def get_file_length(f_in: BufferedIOBase) -> int:
pos = f_in.tell()
f_in.seek(0, io.SEEK_END)
length = f_in.tell()
f_in.seek(pos, io.SEEK_SET)
return length
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
super().parse(data_in, conf)
header: Model3DHeader = kwargs['header']
is_world_model_3d: bool = kwargs['is_world_model_3d']
def parse_vertices_normals(mode: int):
res = []
group = []
for i in range(header.n_vertices):
# Vertices
xyzi = (int.from_bytes(data_in.read(2), 'little', signed=True),
int.from_bytes(data_in.read(2), 'little', signed=True),
int.from_bytes(data_in.read(2), 'little', signed=True),
int.from_bytes(data_in.read(2), 'little'))
group.append(xyzi[:3])
if xyzi[3] < 1:
error_cause = \
NegativeIndexError.CAUSE_VERTEX if mode == 0 else NegativeIndexError.CAUSE_VERTEX_NORMAL
raise NegativeIndexError(data_in.tell(), error_cause,
xyzi[3], xyzi)
elif xyzi[3] == 1:
res.append(np.array(group, dtype=np.int16))
group = []
if group:
res.append(np.array(group, dtype=np.int16))
return res, len(res)
vertices, n_vertices_groups = parse_vertices_normals(0)
normals: List[np.ndarray] = []
if not (is_world_model_3d
and conf.game in (G.HARRY_POTTER_1_PS1, G.HARRY_POTTER_2_PS1)):
normals, n_normals_groups = parse_vertices_normals(1)
if n_vertices_groups != n_normals_groups:
raise VerticesNormalsGroupsMismatch(n_vertices_groups,
n_normals_groups,
data_in.tell())
# Faces
quads = []
tris = []
faces_normals = []
faces_texture_ids: List[int] = []
if conf.game == G.CROC_2_DEMO_PS1_DUMMY or not is_world_model_3d: # Large face headers (Actors' models)
for face_id in range(header.n_faces):
raw_face_data = (int.from_bytes(data_in.read(2),
'little',
signed=True),
int.from_bytes(data_in.read(2),
'little',
signed=True),
int.from_bytes(data_in.read(2),
'little',
signed=True),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'))
if raw_face_data[
9] & 0x0800: # 1st vertex, then 2nd, 4th and 3rd, except in Croc 2 Demo Dummy WADs
if conf.game != G.CROC_2_DEMO_PS1_DUMMY:
# FIXME
# quads.append((raw_face_data[4], raw_face_data[5], raw_face_data[7], raw_face_data[6]))
quads.append((raw_face_data[4], raw_face_data[5],
raw_face_data[6], raw_face_data[7]))
else:
quads.append((raw_face_data[4], raw_face_data[5],
raw_face_data[6], raw_face_data[7]))
else: # 1st vertex, then 2nd and 3rd
tris.append(
(raw_face_data[4], raw_face_data[5], raw_face_data[6]))
faces_normals.append(raw_face_data[:3])
faces_texture_ids.append(raw_face_data[8])
if raw_face_data[3] < 1:
raise NegativeIndexError(data_in.tell(),
NegativeIndexError.CAUSE_FACE,
raw_face_data[3], raw_face_data)
else: # Small face headers (Subchunks' models)
for face_id in range(header.n_faces):
raw_face_data = (int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'),
int.from_bytes(data_in.read(2), 'little'))
if raw_face_data[5] & 0x0800:
quads.append((raw_face_data[0], raw_face_data[1],
raw_face_data[2], raw_face_data[3]))
else:
tris.append(
(raw_face_data[0], raw_face_data[1], raw_face_data[2]))
faces_texture_ids.append(raw_face_data[4])
quads = np.array(quads, dtype=np.uint16)
tris = np.array(tris, dtype=np.uint16)
faces_normals = np.array(faces_normals, dtype=np.int16)
if conf.game in (G.CROC_2_PS1, G.CROC_2_DEMO_PS1,
G.CROC_2_DEMO_PS1_DUMMY):
bounding_box_info_size = 44
else: # Harry Potter 1 & 2
bounding_box_info_size = 32
data_in.seek(header.n_bounding_box_info * bounding_box_info_size,
SEEK_CUR)
return cls(header, is_world_model_3d, vertices, normals, quads, tris,
faces_normals, faces_texture_ids, n_vertices_groups)
