def _store_file(self, fobj: io.BufferedIOBase) -> typing.Iterable[bytes]:
need_compression = self._comp_cfg is not None
orig_size = None
if need_compression and fobj.seekable():
orig_size = self._get_size_till_eof(fobj)
if orig_size < self._comp_cfg.min_size:
need_compression = False
need_encryption = self._enc_cfg is not None
yield self._make_header(comp=need_compression, enc=need_encryption)
chunks = self._iter_file_chunk(fobj, self._chunk_size)
if need_compression:
chunks = self._compress(chunks, orig_size)
if need_encryption:
chunks = self._encrypt(chunks)
yield from chunks
def __init__(self, stream: io.BufferedIOBase):
if not isinstance(stream, io.BufferedIOBase):
raise TypeError("need BufferedIOBase")
self.stream = stream
assert stream.isatty() is False and stream.seekable() is True
def __init__(self, stream: io.BufferedIOBase, new: Version = None):
"""Parse a MIX from `stream`, which must be a buffered file object.
If `new` is not None, initialize an empty MIX of this version instead.
MixParseError is raised on parsing errors.
"""
# Initialize mandatory attributes
self._dirty = False
self._stream = None
self._open = []
# If stream is, for example, a raw I/O object, files could be destroyed
# without ever raising an error, so check this.
if not isinstance(stream, io.BufferedIOBase):
raise TypeError("`stream` must be an instance of io.BufferedIOBase")
if not stream.readable():
raise ValueError("`stream` must be readable")
if not stream.seekable():
raise ValueError("`stream` must be seekable")
if new is not None:
# Start empty (new file)
if type(new) is not Version:
raise TypeError("`new` must be a Version enumeration member or None")
if version is Version.RG:
raise NotImplementedError("RG MIX files are not yet supported")
self._stream = stream
self._index = {}
self._contents = []
self._version = version
self._flags = 0
return
# Parse an existing file
filesize = stream.seek(0, io.SEEK_END)
if filesize <= 6:
raise MixParseError("File too small")
stream.seek(0)
first4 = stream.read(4)
if first4 == b"MIX1":
raise NotImplementedError("RG MIX files are not yet supported")
elif first4[:2] == b"\x00\x00":
# It seems we have a RA or TS MIX so check the flags
flags = int.from_bytes(first4[2:], "little")
if flags > 3:
raise MixParseError("Unsupported properties")
if flags & 2:
raise NotImplementedError("Encrypted MIX files are not yet supported")
# FIXME HERE: 80 bytes of westwood key_source if encrypted,
# to create a 56 byte long blowfish key from it.
#
# They are followed by a number of 8 byte blocks,
# the first of them decrypting to filecount and bodysize.
# Encrypted TS MIXes have a key.ini we can check for later,
# so at this point assume Version.TS only if unencrypted.
# Stock RA MIXes seem to be always encrypted.
version = Version.TS
# RA/TS MIXes hold their filecount after the flags,
# whilst for TD MIXes their first two bytes are the filecount.
filecount = int.from_bytes(stream.read(2), "little")
else:
version = Version.TD
flags = 0
filecount = int.from_bytes(first4[:2], "little")
stream.seek(2)
# From here it's the same for every unencrypted MIX
bodysize = int.from_bytes(stream.read(4), "little")
indexoffset = stream.tell()
indexsize = filecount * 12
bodyoffset = indexoffset + indexsize
# Check if data is sane
# FIXME: Checksummed MIXes have 20 additional bytes after the body.
if filesize - bodyoffset != bodysize:
raise MixParseError("Incorrect filesize or invalid header")
# OK, time to read the index
index = {}
for key, offset, size in struct.iter_unpack("<LLL", stream.read(indexsize)):
offset += bodyoffset
if offset + size > filesize:
raise MixParseError("Content extends beyond end of file")
index[key] = _MixNode(key, offset, size, size, None)
if len(index) != filecount:
raise MixParseError("Duplicate key")
# Now read the names
# TD/RA: 1422054725; TS: 913179935
for dbkey in (1422054725, 913179935):
if dbkey in index:
stream.seek(index[dbkey].offset)
header = stream.read(32)
if header != b"XCC by Olaf van der Spek\x1a\x04\x17'\x10\x19\x80\x00":
continue
dbsize = int.from_bytes(stream.read(4), "little") # Total filesize
if dbsize != index[dbkey].size or dbsize > 16777216:
raise MixParseError("Invalid name table")
# Skip four bytes for XCC type; 0 for LMD, 2 for XIF
# Skip four bytes for DB version; Always zero
stream.seek(8, io.SEEK_CUR)
gameid = int.from_bytes(stream.read(4), "little") # XCC Game ID
# XCC saves alias numbers, so converting them
# to `Version` is not straight forward.
# FIXME: Check if Dune games and Nox also use MIX files
if gameid == 0:
if version is not Version.TD:
continue
elif gameid == 1:
version = Version.RA
elif 2 <= gameid <= 6 or gameid == 15:
version = Version.TS
else:
continue
namecount = int.from_bytes(stream.read(4), "little")
bodysize = dbsize - 53 # Size - Header - Last byte
namelist = stream.read(bodysize).split(b"\x00") if bodysize else []
if len(namelist) != namecount:
raise MixParseError("Invalid name table")
# Remove Database from index
del index[dbkey]
# Add names to index
names = False
for name in namelist:
name = name.decode(ENCODING, "ignore")
key = genkey(name, version)
if key in index:
index[key].name = name
names = True
# XCC sometimes puts two Databases in a file by mistake,
# so if no names were found, give it another try
if names: break
# Create a sorted list of all contents
contents = sorted(index.values(), key=lambda node: node.offset)
# Calculate alloc values
# This is the size up to wich a file may grow without needing a move
for i in range(len(contents) - 1):
contents[i].alloc = contents[i+1].offset - contents[i].offset
if contents[i].alloc < contents[i].size:
raise MixParseError("Overlapping file boundaries")
# Populate the object
self._stream = stream
self._version = version
self._index = index
self._contents = contents
self._flags = flags