class Object(object): Type = ObjectType Properties = ObjectProperties Identifier = ObjectIdentifier version_format = "<I" version_parser = FileStruct(version_format) valid_version = 119 # First 2 bytes decide which constructor(?) to use, 1 for .mob files, -1 for .pro files # There is also something called obj dif file, probably patch related. constructor_type_format = "H" # .mob files: unknown_data_format = "30s" raw_identifier_format = "16s" # matches file name, unique per entity per map raw_type_format = "I" # If that does not fail reads 2 more bytes full_format = "<" + "".join((constructor_type_format, unknown_data_format, raw_identifier_format, raw_type_format)) full_parser = FileStruct(full_format) def __init__(self, version: int, type: ObjectType, identifier: ObjectIdentifier, properties: ObjectProperties): self.version = version self.type = type self.identifier = identifier @classmethod def read_from(cls, obj_file: io.FileIO) -> "Object": version, = cls.version_parser.unpack_from_file(obj_file) if (version != cls.valid_version): raise TypeError("Arkanum does not support object version %d" % version) constructor, unknown_data, raw_identifier, raw_type = cls.full_parser.unpack_from_file( obj_file) type = Object.Type(raw_type) properties = Object.Properties.read_from(obj_file, obj_type=type) return Object(version=version, type=type, identifier=Object.Identifier(raw_identifier), properties=properties) def write_to(self, obj_file: io.FileIO) -> None: raise NotImplementedError()
def read(cls, sector_blocked_file_path: str) -> "SectorBlockades": with open(sector_blocked_file_path, "rb") as sector_blocked_file: length, = cls.length_parser.unpack_from_file(sector_blocked_file) raw_blocked_sectors_parser = FileStruct("<%d%s" % (length, cls.data_format)) raw_blocked_sectors = raw_blocked_sectors_parser.unpack_from_file(sector_blocked_file) # Convert raw to real. blocked_sectors = [] for raw_blocked_sector in raw_blocked_sectors: blocked_sectors.append((raw_blocked_sector & 0xFFF, raw_blocked_sector >> 26)) return BlockedSectors(file_path=sector_blocked_file_path, blocked_sectors=blocked_sectors)
class DatFooter(object): class Constants(object): marker = b"1TAD" guid_format = "16s" marker_format = "4s" # Should always equal "1TAD" ? file_names_size_format = "I" # Total allocation size for file names (we will probably ignore this) footer_plus_entries_size_format = "I" full_format = "<" + guid_format + marker_format + file_names_size_format + footer_plus_entries_size_format parser = FileStruct(full_format) def __init__(self, guid: bytes, marker: bytes, file_names_size: int, footer_plus_entries_size: int): assert marker == self.Constants.marker self.guid = guid self.file_names_size = file_names_size self.footer_plus_entries_size = footer_plus_entries_size @classmethod def read_from(cls, dat_file: io.FileIO) -> "DatFooter": guid, marker, file_names_size, footer_plus_entries_size = DatFooter.parser.unpack_from_file( dat_file) return DatFooter(guid=guid, marker=marker, file_names_size=file_names_size, footer_plus_entries_size=footer_plus_entries_size) @property def marker(self) -> str: return self.Constants.marker
class DatEntry(object): class Flags(object): is_compressed = 0x002 is_directory = 0x400 padding_format = "4x" # Its always saved as zero, but in memory its usually a 32bit pointer to the name. flags_format = "I" full_size_format = "I" compressed_size_format = "I" location_format = "I" full_format = "<" + padding_format + flags_format + full_size_format + compressed_size_format + location_format parser = FileStruct(full_format) def __init__(self, flags: int, full_size: int, compressed_size: int, location: int): self.is_compressed = bool(flags & self.Flags.is_compressed) self.is_directory = bool(flags & self.Flags.is_directory) self.full_size = full_size self.compressed_size = compressed_size self.location = location @classmethod def read_from(cls, dat_file: io.FileIO) -> "DatEntry": flags, full_size, compressed_size, location = DatEntry.parser.unpack_from_file( dat_file) return DatEntry(flags=flags, full_size=full_size, compressed_size=compressed_size, location=location)
class SectorBlockades(object): """ Binary file format: - Blocked (1 bit) * 4096 """ raw_blockades_format = "<512B" raw_blockades_parser = FileStruct(raw_blockades_format) def __init__(self, raw_blockades: List[Any]): self.raw_blockades = raw_blockades def __len__(self) -> int: return len(self.raw_blockades) def __getitem__(self, index: int) -> int: return self.raw_blockades[index] @classmethod def read_from(cls, sector_file: io.FileIO) -> "SectorBlockades": raw_blockades = cls.raw_blockades_parser.unpack_from_file(sector_file) return SectorBlockades(raw_blockades=raw_blockades) def write_to(self, sector_file: io.FileIO) -> None: self.raw_blockades_parser.pack_into_file(sector_file, self.raw_blockades)
def write(self, sector_blocked_file_path: str) -> None: with open(sector_blocked_file_path, "wb") as sector_blocked_file: length = len(self) if (length == 0): return self.length_parser.pack_into_file(sector_blocked_file, length) # Convert real to raw. raw_blocked_sectors = [] for sector_x, sector_y in self: raw_blocked_sectors.append(sector_x | (sector_y << 26)) raw_blocked_sectors_parser = FileStruct("<%d%s" % (length, self.data_format)) raw_blocked_sectors_parser.pack_into_file(sector_blocked_file, *raw_blocked_sectors)
def write_to(self, sector_file: io.FileIO) -> None: count = len(self.raw_lights) self.count_parser.pack_into_file(sector_file, count) if (count > 0): fmt = "<" + self.raw_format * count FileStruct(fmt).pack_into_file(sector_file, *self.raw_lights)
class SectorRoofs(object): """ Binary file format: - Roof (4 bytes) * 256 """ type_format = "<I" raw_roofs_format = "<256I" type_parser = FileStruct(type_format) raw_roofs_parser = FileStruct(raw_roofs_format) def __init__(self, type: int, raw_roofs: List[Any]): self.type = type self.raw_roofs = raw_roofs def __len__(self) -> int: return len(self.raw_roofs) def __getitem__(self, index: int) -> int: return self.raw_roofs[index] @classmethod def read_from(cls, sector_file: io.FileIO) -> "SectorRoofs": type, = cls.type_parser.unpack_from_file(sector_file) if type == 0: raw_roofs = cls.raw_roofs_parser.unpack_from_file(sector_file) else: raw_roofs = [] return SectorRoofs(type=type, raw_roofs=raw_roofs) def write_to(self, sector_file: io.FileIO) -> None: self.type_parser.pack_into_file(sector_file, self.type) if self.type == 0: self.raw_roofs_parser.pack_into_file(sector_file, *self.raw_roofs)
def read(cls, sector_blocked_file_path: str) -> "SectorBlockades": with open(sector_blocked_file_path, "rb") as sector_blocked_file: length, = cls.length_parser.unpack_from_file(sector_blocked_file) raw_blocked_sectors_parser = FileStruct("<%d%s" % (length, cls.data_format)) raw_blocked_sectors = raw_blocked_sectors_parser.unpack_from_file( sector_blocked_file) # Convert raw to real. blocked_sectors = [] for raw_blocked_sector in raw_blocked_sectors: blocked_sectors.append( (raw_blocked_sector & 0xFFF, raw_blocked_sector >> 26)) return BlockedSectors(file_path=sector_blocked_file_path, blocked_sectors=blocked_sectors)
def read_from(cls, sector_file: io.FileIO) -> "SectorLights": count, = cls.count_parser.unpack_from_file(sector_file) if (count > 0): fmt = "<" + cls.raw_format * count raw_lights = FileStruct(fmt).unpack_from_file(sector_file) else: raw_lights = [] return SectorLights(raw_lights)
class SectorTileScripts(object): """ Binary file format: - Tile scripts count (4 bytes) - Tile script (24 bytes) * count Binary file format per tile script: - ? (4 bytes) - Index of tile in sector (2 bytes) - ? (2 bytes) - Flags (4 bytes) - Counters (4 bytes) - Script ID (4 bytes) - ? (4 bytes) """ count_format = "<I" raw_format = "24s" count_parser = FileStruct(count_format) def __init__(self, raw_scripts: List[Any] = []): self.raw_scripts = raw_scripts def __len__(self) -> int: return len(self.raw_scripts) def __getitem__(self, index: int) -> int: return self.raw_scripts[index] @classmethod def read_from(cls, sector_file: io.FileIO) -> "SectorTileScripts": count, = cls.count_parser.unpack_from_file(sector_file) if (count > 0): fmt = "<" + cls.raw_format * count raw_scripts = FileStruct(fmt).unpack_from_file(sector_file) else: raw_scripts = [] return SectorTileScripts(raw_scripts=raw_scripts) def write_to(self, sector_file: io.FileIO) -> None: count = len(self.raw_scripts) count_data = self.count_parser.pack_into_file(sector_file, count) if (count > 0): fmt = "<" + self.raw_format * count raw_data = FileStruct(fmt).pack_into_file(sector_file, *self.raw_scripts)
class SectorObjects(object): length_format = "I" length_parser = FileStruct("<" + length_format) def __init__(self, objects: List[Any] = []): self.objects = objects def __len__(self): return len(self.objects) def __getitem__(self, index: int) -> Object: return self.objects[index] def __iter__(self): return iter(self.objects) @classmethod def read_from(cls, sector_file: io.FileIO) -> "SectorInfo": # Save current position in file tell = sector_file.tell() # Go to end of file minus size of length. sector_file.seek(-cls.length_parser.size, 2) length, = cls.length_parser.unpack_from_file(sector_file) print(length) objects = [] if length: # Go back to saved position sector_file.seek(tell) for _ in range(length): objects.append(Object.read_from(sector_file)) return SectorObjects(objects=objects) def write_to(self, sector_file: io.FileIO) -> None: for obj in self: obj.write_to(sector_file) self.length_parser.pack_into_file(sector_file, len(self))
class SectorTiles(object): """ Binary file format: - Tile (4 bytes) * 4096 Binary file format per tile (old notes: might be little or big endian): First byte: - Flipped (1 bit) - Unknown (4 bits) - Flippable (2 bits) Second byte: - Outdoor (1 bit) - Variant (3 bits) (tile file name ending with a, b, ..., or h) - Rotation (4 bits) Third / Fourth byte: - Some index (6 bits) # Probably indicates the art file prefix to use. - Some index (6 bits) # In my notes its stated to always be the same as the former, # but its probably the other terrain in a transition. - Unknown (4 bits) """ raw_tiles_format = "<4096I" raw_tiles_parser = FileStruct(raw_tiles_format) def __len__(self) -> int: return len(self.raw_tiles) def __getitem__(self, index: int) -> int: return self.raw_tiles[index] def __init__(self, raw_tiles: List[Any]): self.raw_tiles = raw_tiles @classmethod def read_from(cls, sector_file: io.FileIO) -> "SectorTiles": raw_tiles = cls.raw_tiles_parser.unpack_from_file(sector_file) return SectorTiles(raw_tiles) def write_to(self, sector_file: io.FileIO) -> None: raw_tiles_data = self.raw_tiles_parser.pack_into_file( sector_file, *self.raw_tiles)
class MapProperties(object): # This is the original map type (The type of the tiles the map was created with), here it is saved in 4 bytes # And in terrain.tdf it is saved as 8 bytes as well for some reason. # This value seems to have little impact (I didn't find yet what uses it, since so far everything i saw used data # directly from the sectors descriptors) # In 'arcanum1.dat' under 'terrain/forest to snowy plains' there is actually a mismatch with terrain.tdf (That is # the only one) so i assume that the value in the here is more important (since the tdf value is the wrong one). # The values here fit the values in 'arcanum1.dat' under 'terrain/terrain.mes' original_type_format = "I" # This seems to be some kind of a computer stamps, and it seems to only server as informational value. # When creating maps with worldEd The whole value can change between computers, on one of my computers the value # also change a bit each restart (Only the second byte) on the other the number is always consistent. stamp_format = "I" tile_rows_format = "Q" tile_cols_format = "Q" full_format = "<" + original_type_format + stamp_format + tile_rows_format + tile_cols_format parser = FileStruct(full_format) def __init__(self, file_path: str, original_type: int, stamp: int, tile_rows: int, tile_cols: int): self.file_path = file_path self.original_type = original_type self.stamp = stamp self.tile_rows = tile_rows self.tile_cols = tile_cols @classmethod def read(cls, map_properties_file_path: str) -> "MapProperties": with open(map_properties_file_path, "rb") as map_properties_file: original_type, stamp, tile_rows, tile_cols = cls.parser.unpack_from_file( map_properties_file) return MapProperties(file_path=map_properties_file_path, original_type=original_type, stamp=stamp, tile_rows=tile_rows, tile_cols=tile_cols)
class SectorLights(object): """ Binary file format: - Light count (4 bytes) - Light (48 bytes) * Light count """ count_format = "<I" raw_format = "48s" count_parser = FileStruct(count_format) def __init__(self, raw_lights: List[Any]): self.raw_lights = raw_lights def __len__(self) -> int: return len(self.raw_lights) def __getitem__(self, index: int) -> int: return self.raw_lights[index] @classmethod def read_from(cls, sector_file: io.FileIO) -> "SectorLights": count, = cls.count_parser.unpack_from_file(sector_file) if (count > 0): fmt = "<" + cls.raw_format * count raw_lights = FileStruct(fmt).unpack_from_file(sector_file) else: raw_lights = [] return SectorLights(raw_lights) def write_to(self, sector_file: io.FileIO) -> None: count = len(self.raw_lights) self.count_parser.pack_into_file(sector_file, count) if (count > 0): fmt = "<" + self.raw_format * count FileStruct(fmt).pack_into_file(sector_file, *self.raw_lights)
class BlockedSectors(Sequence): """ If a sector is present in this object it is "blocked on the world map". """ length_format = "<I" length_parser = FileStruct(length_format) data_format = "Q" def __init__(self, file_path: str, blocked_sectors: List[Tuple[int, int]] = []): self.blocked_sectors = blocked_sectors self.file_path = file_path def __getitem__(self, index: int) -> Tuple[int, int]: return self.blocked_sectors[index] def __len__(self) -> int: return len(self.blocked_sectors) @classmethod def read(cls, sector_blocked_file_path: str) -> "SectorBlockades": with open(sector_blocked_file_path, "rb") as sector_blocked_file: length, = cls.length_parser.unpack_from_file(sector_blocked_file) raw_blocked_sectors_parser = FileStruct("<%d%s" % (length, cls.data_format)) raw_blocked_sectors = raw_blocked_sectors_parser.unpack_from_file( sector_blocked_file) # Convert raw to real. blocked_sectors = [] for raw_blocked_sector in raw_blocked_sectors: blocked_sectors.append( (raw_blocked_sector & 0xFFF, raw_blocked_sector >> 26)) return BlockedSectors(file_path=sector_blocked_file_path, blocked_sectors=blocked_sectors) def write(self, sector_blocked_file_path: str) -> None: with open(sector_blocked_file_path, "wb") as sector_blocked_file: length = len(self) if (length == 0): return self.length_parser.pack_into_file(sector_blocked_file, length) # Convert real to raw. raw_blocked_sectors = [] for sector_x, sector_y in self: raw_blocked_sectors.append(sector_x | (sector_y << 26)) raw_blocked_sectors_parser = FileStruct("<%d%s" % (length, self.data_format)) raw_blocked_sectors_parser.pack_into_file(sector_blocked_file, *raw_blocked_sectors)
class Dat(object): number_of_entries_parser = FileStruct("<I") file_name_length_parser = FileStruct("<I") def __init__(self, dat_file: io.FileIO, footer: DatFooter, name_to_entry: Dict[str, DatEntry]): self.dat_file = dat_file self.footer = footer self.name_to_entry = name_to_entry @classmethod def open(cls, dat_file_path: str) -> "Dat": dat_file = open(dat_file_path, "rb") dat_file.seek(-DatFooter.parser.size, io.SEEK_END) footer = DatFooter.read_from(dat_file) dat_file.seek(-footer.footer_plus_entries_size, io.SEEK_END) number_of_entries, = cls.number_of_entries_parser.unpack_from_file( dat_file) name_to_entry = OrderedDict() # type: Dict[str, DatEntry] for _ in range(number_of_entries): file_name_length, = cls.file_name_length_parser.unpack_from_file( dat_file) file_name = dat_file.read( file_name_length - 1).decode() # we don't want to save the last null dat_file.read(1) # to skip the null byte name_to_entry[file_name] = DatEntry.read_from(dat_file) return Dat(dat_file=dat_file, footer=footer, name_to_entry=name_to_entry) def __contains__(self, name: str) -> bool: return name in self.name_to_entry def __getitem__(self, name: str) -> bytes: entry = self.name_to_entry[name] if entry.is_directory: return b"" self.dat_file.seek(entry.location, io.SEEK_SET) raw_data = self.dat_file.read(entry.compressed_size) if entry.is_compressed: return zlib.decompress(raw_data) else: return raw_data def keys(self) -> Iterable[str]: return self.name_to_entry.keys()
class ObjectProperties(object): flags_parsers = ( FileStruct("<H"), # 0 FileStruct("<H4B"), # 1 FileStruct("<H8B"), # 2 FileStruct("<H12B"), # 3 FileStruct("<H16B"), # 4 FileStruct("<H20B") # 5 ) # Mapping from type to tuple of all (field name, parser) type_fields = (Fields.wall_fields, Fields.portal_fields, Fields.container_fields, Fields.scenery_fields, Fields.projectile_fields, Fields.weapon_fields, Fields.ammo_fields, Fields.armor_fields, Fields.gold_fields, Fields.food_fields, Fields.scroll_fields, Fields.key_fields, Fields.keyring_fields, Fields.written_fields, Fields.generic_fields, Fields.player_fields, Fields.critter_fields, Fields.trap_fields) # Size in bytes of included field flags. # len(Fields.type) / 32 type_flags_length = ( 3, # 00 Wall 3, # 01 Portal 3, # 02 Container 3, # 03 Scenery 0, # 04 Projectile 4, # 05 Weapon 4, # 06 Ammo 4, # 07 Armor 4, # 08 Gold 4, # 09 Food 4, # 10 Scroll 4, # 11 Key 4, # 12 KeyRing 4, # 13 Written 4, # 14 Generic 5, # 15 Player 5, # 16 Critter 3 # 17 Trap ) @classmethod def read_from(cls, obj_file: io.FileIO, obj_type: ObjectType = None) -> "ObjectProperties": raise NotImplementedError() field_count, *raw_flags = cls.flags_parsers[ cls.type_flags_length[obj_type]].unpack_from_file(obj_file) # Bytes to bit array. flags = np.fliplr( np.unpackbits(np.array(raw_flags, dtype=np.uint8)).reshape(-1, 8)).flatten() if (field_count != np.sum(flags)): raise RuntimeError( "Field count doesn't match actual: %d versus %d" % (field_count, np.sum(flags))) # Parse fields from file raw_fields = {} for index in np.nonzero(flags)[0]: name, parse_func = cls.type_fields[obj_type][index] print(name) raw_fields[name] = parse_func(obj_file)
class SectorInfo(object): """ Binary file format: - Type (4 bytes) - Info (varying bytes) type 0: - Nothing type 1: - Script tile count (4 bytes) # List of tile scripts (n * 24 bytes) type 2: - all of type 1 - Sector Script Flags (4 bytes) - Sector Script Counters (4 * 1 byte) - Sector Script ID (4 bytes) type 3: - all of type 2 - Town Map ID (4 bytes) - Magick/Tech aptitude (4 bytes signed) - Light Scheme (4 bytes) - NULL (4 bytes) - Music ID (4 bytes) - Ambient ID (4 bytes) type 4: (Default, WorldEd always(?) saves as type 4) - all of type 3 - Blocked tile array (64 * 64 bits) """ # Figure out whether unchanging bits have meaning class Type(object): NO_INFO = 0x00AA0000 TILE_SCRIPTS = 0x00AA0001 ALL_SCRIPTS = 0x00AA0002 BASIC = 0x00AA0003 FULL = 0x00AA0004 type_format = "<I" type_parser = FileStruct(type_format) script_flags_format = "I" script_counters_format = "I" script_id_format = "I" sector_script_format = script_flags_format + script_counters_format + script_id_format town_map_format = "I" magick_aptitude_format = "i" light_scheme_format = "I" unknown_format = "I" # Always 0 music_format = "I" ambient_format = "I" blockades_format = "512B" basic_format = (town_map_format + magick_aptitude_format + light_scheme_format + unknown_format + music_format + ambient_format) sector_script_parser = FileStruct("<" + sector_script_format) basic_parser = FileStruct("<" + sector_script_format + basic_format) full_parser = FileStruct("<" + sector_script_format + basic_format + blockades_format) def __init__(self, type: int = Type.FULL, tile_scripts: SectorTileScripts = None, sector_script: (int, int, int) = (0, 0, 0), town_map: int = 0, magick_aptitude: int = 0, light_scheme: int = 0, music: int = 0, ambient: int = 0, blockades: List[Any] = []): self.type = type self.tile_scripts = tile_scripts self.sector_script = sector_script self.town_map = town_map self.magick_aptitude = magick_aptitude self.light_scheme = light_scheme self.music = music self.ambient = ambient self.blockades = blockades def __len__(self) -> int: return len(self.raw_blockades) def __getitem__(self, index: int) -> int: return self.raw_blockades[index] @classmethod def read_from(cls, sector_file: io.FileIO) -> "SectorInfo": type, = cls.type_parser.unpack_from_file(sector_file) if type == cls.Type.NO_INFO: return SectorInfo(type=type) tile_scripts = SectorTileScripts.read_from(sector_file) if type == cls.Type.TILE_SCRIPTS: return SectorInfo(type=type, tile_scripts=tile_scripts) elif type == cls.Type.ALL_SCRIPTS: sector_script = cls.sector_script_parser.unpack_from_file( sector_file) return SectorInfo(type=type, tile_scripts=tile_scripts, sector_script=sector_script) elif type == cls.Type.BASIC: (sector_script_flags, sector_script_counters, sector_script_id, town_map, magick_aptitude, light_scheme, _, music, ambient) = cls.basic_parser.unpack_from_file(sector_file) sector_script = (sector_script_flags, sector_script_counters, sector_script_id) return SectorInfo(type=type, tile_scripts=tile_scripts, sector_script=sector_script, town_map=town_map, magick_aptitude=magick_aptitude, light_scheme=light_scheme, music=music, ambient=ambient) elif type == cls.Type.FULL: (sector_script_flags, sector_script_counters, sector_script_id, town_map, magick_aptitude, light_scheme, _, music, ambient, *blockades) = cls.full_parser.unpack_from_file(sector_file) sector_script = (sector_script_flags, sector_script_counters, sector_script_id) return SectorInfo(type=type, tile_scripts=tile_scripts, sector_script=sector_script, town_map=town_map, magick_aptitude=magick_aptitude, light_scheme=light_scheme, music=music, ambient=ambient, blockades=blockades) else: raise NotImplementedError("Can not handle unknown type %d" % (type)) def write_to(self, sector_file: io.FileIO) -> None: self.type_parser.pack_into_file(sector_file, self.type) if self.type == self.Type.NO_INFO: return self.tile_scripts.write_to(sector_file) if self.type == self.Type.ALL_SCRIPTS: self.sector_script_parser.pack_into_file(sector_file, *self.sector_script) elif self.type == self.Type.BASIC: self.basic_parser.pack_into_file(sector_file, *self.sector_script, self.town_map, self.magick_aptitude, self.light_scheme, 0, self.music, self.ambient) elif self.type == self.Type.FULL: self.full_parser.pack_into_file(sector_file, *self.sector_script, self.town_map, self.magick_aptitude, self.light_scheme, 0, self.music, self.ambient, *self.blockades)