def mcaIterator(mca_path, mca_filename): registry = OpaqueRegistry(64) # log2(max block ID) with RegionFile(mca_path + mca_filename) as region_file: for chunk_x in range(0, 32): for chunk_z in range(0, 32): try: chunk = region_file.load_chunk(chunk_x, chunk_z) sections = chunk.value[''].value['Level'].value[ 'Sections'].value except (KeyError, ValueError, BufferUnderrun) as e: if type(e) is BufferUnderrun: print("Failed loading chunk:", chunk_x, chunk_z, "Reason:", type(e).__name__, e) continue for section in sections: try: blocks = BlockArray.from_nbt(section, registry) except KeyError as e: continue yield { 'mca': mca_filename, 'x': chunk_x, 'z': chunk_z, 'y': int.from_bytes(section.value['Y'].to_bytes(), 'big'), 'blocks': blocks, }
def test_chunk_internals(): blocks = BlockArray.empty(OpaqueRegistry(13)) storage = blocks.storage # Accumulate blocks added = [] for i in range(300): blocks[i] = i added.append(i) assert blocks[:i + 1] == added if i < 256: assert len(blocks.palette) == i + 1 if i < 16: assert storage.value_width == 4 elif i < 32: assert storage.value_width == 5 elif i < 64: assert storage.value_width == 6 elif i < 128: assert storage.value_width == 7 else: assert storage.value_width == 8 else: assert blocks.palette == [] assert storage.value_width == 13 # Zero the first 100 blocks for i in range(100): blocks[i] = 0 blocks.repack() assert len(blocks.palette) == 201 assert storage.value_width == 8 # Zero blocks 100-199 for i in range(100, 200): blocks[i] = 0 blocks.repack() assert len(blocks.palette) == 101 assert storage.value_width == 7 # Zero blocks 205 - 300 for i in range(205, 300): blocks[i] = 0 blocks.repack() assert len(blocks.palette) == 6 assert storage.value_width == 4 # Check value for i in range(4096): if 200 <= i < 205: assert blocks[i] == i else: assert blocks[i] == 0
def test_wikivg_example(): # Example from https://wiki.vg/Chunk_Format#Example data = bitstring.BitArray(length=13 * 4096) data[ 0: 64] = '0b0000000000100000100001100011000101001000010000011000100001000001' data[ 64: 128] = '0b0000000100000001100010100111001001100000111101101000110010000111' data = data.bytes blocks = BlockArray.from_bytes(data, 5, OpaqueRegistry(13), []) assert blocks[:24] == [ 1, 2, 2, 3, 4, 4, 5, 6, 6, 4, 8, 0, 7, 4, 3, 13, 15, 16, 9, 14, 10, 12, 0, 2 ]
class Buffer1_7(object): buff = b"" pos = 0 registry = OpaqueRegistry(13) def __init__(self, data=None): if data: self.buff = data def __len__(self): return len(self.buff) - self.pos def add(self, data): """ Add some bytes to the end of the buffer. """ self.buff += data def save(self): """ Saves the buffer contents. """ self.buff = self.buff[self.pos:] self.pos = 0 def restore(self): """ Restores the buffer contents to its state when :meth:`save` was last called. """ self.pos = 0 def discard(self): """ Discards the entire buffer contents. """ self.pos = len(self.buff) def read(self, length=None): """ Read *length* bytes from the beginning of the buffer, or all bytes if *length* is ``None`` """ if length is None: data = self.buff[self.pos:] self.pos = len(self.buff) else: if self.pos + length > len(self.buff): raise BufferUnderrun() data = self.buff[self.pos:self.pos + length] self.pos += length return data def hexdump(self): printable = string.letters + string.digits + string.punctuation data = self.buff[self.pos:] lines = [''] bytes_read = 0 while len(data) > 0: data_line, data = data[:16], data[16:] l_hex = [] l_str = [] for i, c in enumerate(data_line): if PY3: l_hex.append("%02x" % c) c_str = data_line[i:i + 1] l_str.append(c_str if c_str in printable else ".") else: l_hex.append("%02x" % ord(c)) l_str.append(c if c in printable else ".") l_hex.extend([' '] * (16 - len(l_hex))) l_hex.insert(8, '') lines.append("%08x %s |%s|" % (bytes_read, " ".join(l_hex), "".join(l_str))) bytes_read += len(data_line) return "\n ".join(lines + ["%08x" % bytes_read]) # Basic data types -------------------------------------------------------- @classmethod def pack(cls, fmt, *fields): """ Pack *fields* into a struct. The format accepted is the same as for ``struct.pack()``. """ return struct.pack(">" + fmt, *fields) def unpack(self, fmt): """ Unpack a struct. The format accepted is the same as for ``struct.unpack()``. """ fmt = ">" + fmt data = self.read(struct.calcsize(fmt)) fields = struct.unpack(fmt, data) if len(fields) == 1: fields = fields[0] return fields # Blob data types --------------------------------------------------------- @classmethod def pack_blob(cls, fmt, blob): """ Packs a length-prefixed byte string. The *fmt* parameter gives the format of the length prefix. """ return cls.pack(fmt, len(blob)) + blob def unpack_blob(self, fmt): """ Unpacks a length-prefixed byte string. The *fmt* parameter gives the format of the length prefix. """ return self.read(self.unpack(fmt)) @classmethod def pack_varint_blob(cls, blob): """ Packs a length-prefixed byte string. The length prefix is packed as a varint. """ return cls.pack_varint(len(blob)) + blob def unpack_varint_blob(self): """ Unpacks a length-prefixed byte string. The length prefix is unpacked as a varint. """ return self.read(self.unpack_varint()) # Array data types -------------------------------------------------------- @classmethod def pack_array(cls, fmt, array): """ Packs *array* into a struct. The format accepted is the same as for ``struct.pack()``. """ return struct.pack(">" + fmt * len(array), *array) def unpack_array(self, fmt, length): """ Unpack an array struct. The format accepted is the same as for ``struct.unpack()``. """ data = self.read(struct.calcsize(">" + fmt) * length) return list(struct.unpack(">" + fmt * length, data)) # Optional ---------------------------------------------------------------- @classmethod def pack_optional(cls, packer, val): """ Packs a boolean indicating whether *val* is None. If not, ``packer(val)`` is appended to the returned string. """ if val is None: return cls.pack('?', False) else: return cls.pack('?', True) + packer(val) def unpack_optional(self, unpacker): """ Unpacks a boolean. If it's True, return the value of ``unpacker()``. Otherwise return None. """ if self.unpack('?'): return unpacker() else: return None # Varint ------------------------------------------------------------------ @classmethod def pack_varint(cls, number, max_bits=32): """ Packs a varint. """ number_min = -1 << (max_bits - 1) number_max = +1 << (max_bits - 1) if not (number_min <= number < number_max): raise ValueError("varint does not fit in range: %d <= %d < %d" % (number_min, number, number_max)) if number < 0: number += 1 << 32 out = b"" for i in xrange(10): b = number & 0x7F number >>= 7 out += cls.pack("B", b | (0x80 if number > 0 else 0)) if number == 0: break return out def unpack_varint(self, max_bits=32): """ Unpacks a varint. """ number = 0 for i in xrange(10): b = self.unpack("B") number |= (b & 0x7F) << 7 * i if not b & 0x80: break if number & (1 << 31): number -= 1 << 32 number_min = -1 << (max_bits - 1) number_max = +1 << (max_bits - 1) if not (number_min <= number < number_max): raise ValueError("varint does not fit in range: %d <= %d < %d" % (number_min, number, number_max)) return number # Packet ------------------------------------------------------------------ @classmethod def pack_packet(cls, data, compression_threshold=-1): """ Unpacks a packet frame. This method handles length-prefixing and compression. """ if compression_threshold >= 0: # Compress data and prepend uncompressed data length if len(data) >= compression_threshold: data = cls.pack_varint(len(data)) + zlib.compress(data) else: data = cls.pack_varint(0) + data # Prepend packet length return cls.pack_varint(len(data), max_bits=32) + data def unpack_packet(self, cls, compression_threshold=-1): """ Unpacks a packet frame. This method handles length-prefixing and compression. """ body = self.read(self.unpack_varint(max_bits=32)) buff = cls(body) if compression_threshold >= 0: uncompressed_length = buff.unpack_varint() if uncompressed_length > 0: body = zlib.decompress(buff.read()) buff = cls(body) return buff # String ------------------------------------------------------------------ @classmethod def pack_string(cls, text): """ Pack a varint-prefixed utf8 string. """ text = text.encode("utf-8") return cls.pack_varint(len(text), max_bits=16) + text def unpack_string(self): """ Unpack a varint-prefixed utf8 string. """ length = self.unpack_varint(max_bits=16) text = self.read(length).decode("utf-8") return text # JSON -------------------------------------------------------------------- @classmethod def pack_json(cls, obj): """ Serialize an object to JSON and pack it to a Minecraft string. """ return cls.pack_string(json.dumps(obj)) def unpack_json(self): """ Unpack a Minecraft string and interpret it as JSON. """ obj = json.loads(self.unpack_string()) return obj # Chat -------------------------------------------------------------------- @classmethod def pack_chat(cls, message): """ Pack a Minecraft chat message. """ from quarry.types import chat if not isinstance(message, chat.Message): message = chat.Message.from_string(message) return message.to_bytes() def unpack_chat(self): """ Unpack a Minecraft chat message. """ from quarry.types import chat return chat.Message.from_buff(self) # UUID -------------------------------------------------------------------- @classmethod def pack_uuid(cls, uuid): """ Packs a UUID. """ return uuid.to_bytes() def unpack_uuid(self): """ Unpacks a UUID. """ return UUID.from_bytes(self.read(16)) # Position ---------------------------------------------------------------- @classmethod def pack_position(cls, x, y, z): """ Packs a Position. """ def pack_twos_comp(bits, number): if number < 0: number = number + (1 << bits) return number return cls.pack( 'Q', sum((pack_twos_comp(26, x) << 38, pack_twos_comp(12, y) << 26, pack_twos_comp(26, z)))) def unpack_position(self): """ Unpacks a position. """ def unpack_twos_comp(bits, number): if (number & (1 << (bits - 1))) != 0: number = number - (1 << bits) return number number = self.unpack('Q') x = unpack_twos_comp(26, (number >> 38)) y = unpack_twos_comp(12, (number >> 26 & 0xFFF)) z = unpack_twos_comp(26, (number & 0x3FFFFFF)) return x, y, z # Block ------------------------------------------------------------------- @classmethod def pack_block(cls, block, packer=None): """ Packs a block. """ if packer is None: packer = cls.pack_varint return packer(cls.registry.encode_block(block)) def unpack_block(self, unpacker=None): """ Unpacks a block. """ if unpacker is None: unpacker = self.unpack_varint return self.registry.decode_block(unpacker()) # Slot -------------------------------------------------------------------- @classmethod def pack_slot(cls, item=None, count=1, damage=0, tag=None): """ Packs a slot. """ if item is None: return cls.pack('h', -1) item_id = cls.registry.encode('minecraft:item', item) return cls.pack('hbh', item_id, count, damage) + cls.pack_nbt(tag) def unpack_slot(self): """ Unpacks a slot. """ slot = {} item_id = self.unpack('h') if item_id == -1: slot['item'] = None else: slot['item'] = self.registry.decode('minecraft:item', item_id) slot['count'] = self.unpack('b') slot['damage'] = self.unpack('h') slot['tag'] = self.unpack_nbt() return slot # NBT --------------------------------------------------------------------- @classmethod def pack_nbt(cls, tag=None): """ Packs an NBT tag """ if tag is None: # slower but more obvious: # from quarry.types import nbt # tag = nbt.TagRoot({}) return b"\x00" return tag.to_bytes() def unpack_nbt(self): """ Unpacks NBT tag(s). """ from quarry.types import nbt return nbt.TagRoot.from_buff(self) # Entity metadata --------------------------------------------------------- @classmethod def pack_entity_metadata(cls, metadata): """ Packs entity metadata. """ out = b"" for ty_key, val in metadata.items(): ty, key = ty_key out += cls.pack('B', ty << 5 | key) if ty == 0: out += cls.pack('b', val) elif ty == 1: out += cls.pack('h', val) elif ty == 2: out += cls.pack('i', val) elif ty == 3: out += cls.pack('f', val) elif ty == 4: out += cls.pack_string(val) elif ty == 5: out += cls.pack_slot(**val) elif ty == 6: out += cls.pack('iii', *val) elif ty == 7: out += cls.pack_rotation(*val) else: raise ValueError("Unknown entity metadata type: %d" % ty) out += cls.pack('B', 127) return out def unpack_entity_metadata(self): """ Unpacks entity metadata. """ metadata = {} while True: b = self.unpack('B') if b == 127: return metadata ty, key = b >> 5, b & 0x1F if ty == 0: val = self.unpack('b') elif ty == 1: val = self.unpack('h') elif ty == 2: val = self.unpack('i') elif ty == 3: val = self.unpack('f') elif ty == 4: val = self.unpack_string() elif ty == 5: val = self.unpack_slot() elif ty == 6: val = self.unpack('iii') elif ty == 7: val = self.unpack_rotation() else: raise ValueError("Unknown entity metadata type: %d" % ty) metadata[ty, key] = val # Direction --------------------------------------------------------------- @classmethod def pack_direction(cls, direction): """ Packs a direction. """ return cls.pack_varint(directions.index(direction)) def unpack_direction(self): """ Unpacks a direction. """ return directions[self.unpack_varint()] # Rotation ---------------------------------------------------------------- @classmethod def pack_rotation(cls, x, y, z): """ Packs a rotation. """ return cls.pack('fff', x, y, z) def unpack_rotation(self): """ Unpacks a rotation """ return self.unpack('fff')
class Buffer1_13(Buffer1_9): registry = OpaqueRegistry(14) # Chunk section ----------------------------------------------------------- @classmethod def pack_chunk_section_palette(cls, palette): if not palette: return b"" else: return cls.pack_varint(len(palette)) + b"".join( cls.pack_varint(x) for x in palette) def unpack_chunk_section_palette(self, bits): if bits > 8: return [] else: return [self.unpack_varint() for _ in xrange(self.unpack_varint())] # Slot -------------------------------------------------------------------- @classmethod def pack_slot(cls, item=None, count=1, tag=None): """ Packs a slot. """ if item is None: return cls.pack('h', -1) item_id = cls.registry.encode('minecraft:item', item) return cls.pack('hb', item_id, count) + cls.pack_nbt(tag) def unpack_slot(self): """ Unpacks a slot. """ slot = {} item_id = self.unpack('h') if item_id == -1: slot['item'] = None else: slot['item'] = self.registry.decode('minecraft:item', item_id) slot['count'] = self.unpack('b') slot['tag'] = self.unpack_nbt() return slot # Entity metadata --------------------------------------------------------- @classmethod def pack_entity_metadata(cls, metadata): """ Packs entity metadata. """ pack_position = lambda pos: cls.pack_position(*pos) out = b"" for ty_key, val in metadata.items(): ty, key = ty_key out += cls.pack('BB', key, ty) if ty == 0: out += cls.pack('b', val) elif ty == 1: out += cls.pack_varint(val) elif ty == 2: out += cls.pack('f', val) elif ty == 3: out += cls.pack_string(val) elif ty == 4: out += cls.pack_chat(val) elif ty == 5: out += cls.pack_optional(cls.pack_chat, val) elif ty == 6: out += cls.pack_slot(**val) elif ty == 7: out += cls.pack('?', val) elif ty == 8: out += cls.pack_rotation(*val) elif ty == 9: out += cls.pack_position(*val) elif ty == 10: out += cls.pack_optional(pack_position, val) elif ty == 11: out += cls.pack_direction(val) elif ty == 12: out += cls.pack_optional(cls.pack_uuid, val) elif ty == 13: out += cls.pack_block(val) elif ty == 14: out += cls.pack_nbt(val) elif ty == 15: out += cls.pack_particle(*val) else: raise ValueError("Unknown entity metadata type: %d" % ty) out += cls.pack('B', 255) return out def unpack_entity_metadata(self): """ Unpacks entity metadata. """ metadata = {} while True: key = self.unpack('B') if key == 255: return metadata ty = self.unpack('B') if ty == 0: val = self.unpack('b') elif ty == 1: val = self.unpack_varint() elif ty == 2: val = self.unpack('f') elif ty == 3: val = self.unpack_string() elif ty == 4: val = self.unpack_chat() elif ty == 5: val = self.unpack_optional(self.unpack_chat) elif ty == 6: val = self.unpack_slot() elif ty == 7: val = self.unpack('?') elif ty == 8: val = self.unpack_rotation() elif ty == 9: val = self.unpack_position() elif ty == 10: val = self.unpack_optional(self.unpack_position) elif ty == 11: val = self.unpack_direction() elif ty == 12: val = self.unpack_optional(self.unpack_uuid) elif ty == 13: val = self.unpack_block() elif ty == 14: val = self.unpack_nbt() elif ty == 15: val = self.unpack_particle() else: raise ValueError("Unknown entity metadata type: %d" % ty) metadata[ty, key] = val # Particle ---------------------------------------------------------------- @classmethod def pack_particle(cls, id, data=None): """ Packs a particle. """ data = data or {} out = cls.pack_varint(id) if id == 3 or id == 20: out += cls.pack_varint(data['block_state']) elif id == 11: out += cls.pack('ffff', data['red'], data['green'], data['blue'], data['scale']) elif id == 27: out += cls.pack_slot(**data['item']) return out def unpack_particle(self): """ Unpacks a particle. Returns an ``(id, data)`` pair. """ id = self.unpack_varint() if id == 3 or id == 20: data = {'block_state': self.unpack_varint()} elif id == 11: data = dict( zip(('red', 'green', 'blue', 'scale'), self.unpack('ffff'))) elif id == 27: data = {'item': self.unpack_slot()} else: data = {} return id, data # Commands ---------------------------------------------------------------- def unpack_commands(self, resolve_redirects=True): """ Unpacks a command graph. If *resolve_redirects* is ``True`` (the default), the returned structure may contain contain circular references, and therefore cannot be serialized to JSON (or similar). If it is ``False``, all node redirect information is stripped, resulting in a directed acyclic graph. """ # Unpack nodes node_count = self.unpack_varint() nodes = [self.unpack_command_node() for _ in range(node_count)] # Resolve children and redirects for node in nodes: node['children'] = { nodes[idx]['name']: nodes[idx] for idx in node['children'] } if node['redirect'] is not None: if resolve_redirects: node['redirect'] = nodes[node['redirect']] else: node['redirect'] = None return nodes[self.unpack_varint()] def unpack_command_node(self): """ Unpacks a command node. """ node = {} flags = self.unpack('B') node['type'] = ['root', 'literal', 'argument'][flags & 0x03] node['executable'] = bool(flags & 0x04) node['children'] = [ self.unpack_varint() for _ in range(self.unpack_varint()) ] node['redirect'] = self.unpack_varint() if flags & 0x08 else None node['name'] = self.unpack_string() if node['type'] != 'root' else None if node['type'] == 'argument': node['parser'] = self.unpack_string() node['properties'] = self.unpack_command_node_properties( node['parser']) node['suggestions'] = self.unpack_string() if flags & 0x10 else None return node def unpack_command_node_properties(self, parser): """ Unpacks the properties of an ``argument`` command node. """ namespace, parser = parser.split(":", 1) properties = {} if namespace == "brigadier": if parser == "bool": pass elif parser == "string": properties['behavior'] = self.unpack_varint() elif parser in ("double", "float", "integer"): fmt = parser[0] flags = self.unpack('B') properties['min'] = self.unpack(fmt) if flags & 0x01 else None properties['max'] = self.unpack(fmt) if flags & 0x02 else None elif namespace == "minecraft": if parser in ('entity', 'score_holder'): properties['allow_multiple'] = self.unpack('?') elif parser == 'range': properties['allow_decimals'] = self.unpack('?') return properties @classmethod def pack_commands(cls, root_node): """ Packs a command graph. """ # Enumerate nodes nodes = [root_node] idx = 0 while idx < len(nodes): node = nodes[idx] children = list(node['children'].values()) if node['redirect']: children.append(node['redirect']) for child in children: if child not in nodes: nodes.append(child) idx += 1 # Pack nodes out = cls.pack_varint(len(nodes)) for node in nodes: out += cls.pack_command_node(node, nodes) out += cls.pack_varint(nodes.index(root_node)) return out @classmethod def pack_command_node(cls, node, nodes): """ Packs a command node. """ out = b"" flags = (['root', 'literal', 'argument'].index(node['type']) | int(node['executable']) << 2 | int(node['redirect'] is not None) << 3 | int(node['suggestions'] is not None) << 4) out += cls.pack('B', flags) out += cls.pack_varint(len(node['children'])) for child in node['children'].values(): out += cls.pack_varint(nodes.index(child)) if node['redirect'] is not None: out += cls.pack_varint(nodes.index(node['redirect'])) if node['name'] is not None: out += cls.pack_string(node['name']) if node['type'] == 'argument': out += cls.pack_string(node['parser']) out += cls.pack_command_node_properties(node['parser'], node['properties']) if node['suggestions'] is not None: out += cls.pack_string(node['suggestions']) return out @classmethod def pack_command_node_properties(cls, parser, properties): """ Packs the properties of an ``argument`` command node. """ namespace, parser = parser.split(":", 1) out = b"" if namespace == "brigadier": if parser == "bool": pass elif parser == "string": out += cls.pack_varint(properties['behavior']) elif parser in ("double", "float", "integer"): fmt = parser[0] flags = (int(properties['min'] is not None) | int(properties['max'] is not None) << 1) out += cls.pack('B', flags) if properties['min'] is not None: out += cls.pack(fmt, properties['min']) if properties['max'] is not None: out += cls.pack(fmt, properties['max']) elif namespace == "minecraft": if parser in ('entity', 'score_holder'): out += cls.pack('?', properties['allow_multiple']) elif parser == 'range': out += cls.pack('?', properties['allow_decimals']) return out # Recipes ----------------------------------------------------------------- def unpack_recipe(self): """ Unpacks a crafting recipe. """ recipe = {} recipe['name'] = self.unpack_string() recipe['type'] = self.unpack_string() if recipe['type'] == 'crafting_shapeless': recipe['group'] = self.unpack_string() recipe['ingredients'] = [ self.unpack_ingredient() for _ in range(self.unpack_varint()) ] recipe['result'] = self.unpack_slot() elif recipe['type'] == 'crafting_shaped': recipe['width'] = self.unpack_varint() recipe['height'] = self.unpack_varint() recipe['group'] = self.unpack_string() recipe['ingredients'] = [ self.unpack_ingredient() for _ in range(recipe['width'] * recipe['height']) ] recipe['result'] = self.unpack_slot() elif recipe['type'] == 'smelting': recipe['group'] = self.unpack_string() recipe['ingredient'] = self.unpack_ingredient() recipe['result'] = self.unpack_slot() recipe['experience'] = self.unpack('f') recipe['cooking_time'] = self.unpack_varint() return recipe @classmethod def pack_recipe(cls, name, type, **recipe): """ Packs a crafting recipe. """ data = cls.pack_string(name) + cls.pack_string(type) if type == 'crafting_shapeless': data += cls.pack_string(recipe['group']) data += cls.pack_varint(len(recipe['ingredients'])) for ingredient in recipe['ingredients']: data += cls.pack_ingredient(ingredient) data += cls.pack_slot(**recipe['result']) elif type == 'crafting_shaped': data += cls.pack_varint(recipe['width']) data += cls.pack_varint(recipe['height']) data += cls.pack_string(recipe['group']) for ingredient in recipe['ingredients']: data += cls.pack_ingredient(ingredient) data += cls.pack_slot(**recipe['result']) elif type == 'smelting': data += cls.pack_string(recipe['group']) data += cls.pack_ingredient(recipe['ingredient']) data += cls.pack_slot(**recipe['result']) data += cls.pack('f', recipe['experience']) data += cls.pack_varint(recipe['cooking_time']) return data def unpack_ingredient(self): """ Unpacks a crafting recipe ingredient alternation. """ return [self.unpack_slot() for _ in range(self.unpack_varint())] @classmethod def pack_ingredient(cls, ingredient): """ Packs a crafting recipe ingredient alternation. """ data = cls.pack_varint(len(ingredient)) for slot in ingredient: data += cls.pack_slot(**slot) return data