def _save_subchunks_1(self, blocks: "Blocks", palette: AnyNDArray) -> List[Optional[bytes]]: for index, block in enumerate(palette): block: Tuple[Tuple[None, Block], ...] block_data = block[0][1].properties.get("block_data", amulet_nbt.TAG_Int(0)) if isinstance(block_data, amulet_nbt.TAG_Int): block_data = block_data.value # if block_data >= 16: # block_data = 0 else: block_data = 0 palette[index] = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String(block[0][1].namespaced_name), "val": amulet_nbt.TAG_Short(block_data), })) chunk = [] for cy in range(16): if cy in blocks: palette_index, sub_chunk = fast_unique( blocks.get_sub_chunk(cy)) sub_chunk_palette = list(palette[palette_index]) chunk.append(b"\x01" + self._save_palette_subchunk( sub_chunk.ravel(), sub_chunk_palette)) else: chunk.append(None) return chunk
def _decode_blocks( self, chunk_sections: nbt.TAG_List ) -> Tuple[Dict[int, SubChunkNDArray], AnyNDArray]: blocks: Dict[int, SubChunkNDArray] = {} palette = [] palette_len = 0 for section in chunk_sections: cy: int = section["Y"].value section_blocks = numpy.frombuffer( section["Blocks"].value, dtype=numpy.uint8 ) del section["Blocks"] section_data = numpy.frombuffer(section["Data"].value, dtype=numpy.uint8) del section["Data"] section_blocks = section_blocks.reshape((16, 16, 16)) section_blocks = section_blocks.astype(numpy.uint16) section_data = world_utils.from_nibble_array(section_data) section_data = section_data.reshape((16, 16, 16)) if "Add" in section: add_blocks = numpy.frombuffer(section["Add"].value, dtype=numpy.uint8) del section["Add"] add_blocks = world_utils.from_nibble_array(add_blocks) add_blocks = add_blocks.reshape((16, 16, 16)) section_blocks |= add_blocks.astype(numpy.uint16) << 8 # TODO: fix this (section_palette, blocks[cy]) = world_utils.fast_unique( numpy.transpose( (section_blocks << 4) + section_data, (2, 0, 1) ) # YZX -> XYZ ) blocks[cy] += palette_len palette_len += len(section_palette) palette.append(section_palette) if palette: final_palette, lut = numpy.unique( numpy.concatenate(palette), return_inverse=True ) final_palette: numpy.ndarray = numpy.array( [final_palette >> 4, final_palette & 15] ).T for cy in blocks: blocks[cy] = lut[blocks[cy]] else: final_palette = numpy.array([], dtype=numpy.object) return blocks, final_palette
def _decode_blocks( self, chunk: Chunk, chunk_sections: Dict[int, TAG_Compound] ) -> AnyNDArray: blocks: Dict[int, SubChunkNDArray] = {} palette = [] palette_len = 0 for cy, section in chunk_sections.items(): section_blocks = numpy.frombuffer( section.pop("Blocks").value, dtype=numpy.uint8 ) section_data = numpy.frombuffer( section.pop("Data").value, dtype=numpy.uint8 ) section_blocks = section_blocks.reshape((16, 16, 16)) section_blocks = section_blocks.astype(numpy.uint16) section_data = world_utils.from_nibble_array(section_data) section_data = section_data.reshape((16, 16, 16)) if "Add" in section: add_blocks = numpy.frombuffer( section.pop("Add").value, dtype=numpy.uint8 ) add_blocks = world_utils.from_nibble_array(add_blocks) add_blocks = add_blocks.reshape((16, 16, 16)) section_blocks |= add_blocks.astype(numpy.uint16) << 8 # TODO: fix this (section_palette, blocks[cy]) = world_utils.fast_unique( numpy.transpose( (section_blocks << 4) + section_data, (2, 0, 1) ) # YZX -> XYZ ) blocks[cy] += palette_len palette_len += len(section_palette) palette.append(section_palette) if palette: final_palette, lut = numpy.unique( numpy.concatenate(palette), return_inverse=True ) final_palette: numpy.ndarray = numpy.array( [final_palette >> 4, final_palette & 15] ).T for cy in blocks: blocks[cy] = lut[blocks[cy]] else: final_palette = numpy.array([], dtype=object) chunk.blocks = blocks return final_palette
def decode(self, cx: int, cz: int, data: Dict[bytes, bytes]) -> Tuple[Chunk, AnyNDArray]: # chunk_key_base = struct.pack("<ii", cx, cz) chunk = Chunk(cx, cz) chunk_palette = numpy.empty(0, dtype=object) if self.features["terrain"].startswith( "2f"): # ["2farray", "2f1palette", "2fnpalette"] subchunks = [ data.get(b"\x2F" + bytes([i]), None) for i in range(16) ] chunk.blocks, chunk_palette = self._load_subchunks(subchunks) elif self.features["terrain"] == "30array": chunk_data = data.get(b"\x30", None) if chunk_data is not None: block_ids = numpy.frombuffer(chunk_data[:2**15], dtype=numpy.uint8).astype( numpy.uint16) block_data = from_nibble_array( numpy.frombuffer(chunk_data[2**15:2**15 + 2**14], dtype=numpy.uint8)) # there is other data here but we are going to skip over it combined_palette, block_array = fast_unique( numpy.transpose( ((block_ids << 4) + block_data).reshape(16, 16, 128), (0, 2, 1))) chunk.blocks = { i: block_array[:, i * 16:(i + 1) * 16, :] for i in range(8) } palette: AnyNDArray = numpy.array( [combined_palette >> 4, combined_palette & 15]).T chunk_palette = numpy.empty(len(palette), dtype=object) for i, b in enumerate(palette): chunk_palette[i] = ((None, tuple(b)), ) else: raise Exception if self.features["finalised_state"] == "int0-2": if b"\x36" in data: val = struct.unpack("<i", data[b"\x36"])[0] else: val = 2 chunk.status = val if self.features["data_2d"] in [ "height512|biome256", "unused_height512|biome256", ]: d2d = data.get(b"\x2D", b"\x00" * 768) height, biome = d2d[:512], d2d[512:] if self.features["data_2d"] == "height512|biome256": pass # TODO: put this data somewhere chunk.biomes = numpy.frombuffer(biome, dtype="uint8").reshape(16, 16) # TODO: impliment key support # \x2D heightmap and biomes # \x31 block entity # \x32 entity # \x33 ticks # \x34 block extra data # \x35 biome state # \x39 7 ints and an end (03)? Honestly don't know what this is # \x3A fire tick? # \x2E 2d legacy # \x30 legacy terrain # unpack block entities and entities if self.features["block_entities"] == "31list": block_entities = self._unpack_nbt_list(data.get(b"\x31", b"")) chunk.block_entities = self._decode_block_entities(block_entities) if self.features["entities"] == "32list" and amulet.entity_support: entities = self._unpack_nbt_list(data.get(b"\x32", b"")) chunk.entities = self._decode_entities(entities) return chunk, chunk_palette
def _save_subchunks_8(self, blocks: "Blocks", palette: AnyNDArray) -> List[Optional[bytes]]: palette_depth = numpy.array([len(block) for block in palette]) if palette.size: if palette[0][0][0] is None: air = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String("minecraft:air"), "val": amulet_nbt.TAG_Short(0), })) else: air = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String("minecraft:air"), "states": amulet_nbt.TAG_Compound({}), "version": amulet_nbt.TAG_Int(17_629_184), # 1, 13, 0, 0 })) for index, block in enumerate(palette): block: Tuple[Tuple[Optional[int], Block], ...] full_block = [] for sub_block_version, sub_block in block: properties = sub_block.properties if sub_block_version is None: block_data = properties.get("block_data", amulet_nbt.TAG_Int(0)) if isinstance(block_data, amulet_nbt.TAG_Int): block_data = block_data.value # if block_data >= 16: # block_data = 0 else: block_data = 0 sub_block_ = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String( sub_block.namespaced_name), "val": amulet_nbt.TAG_Short(block_data), })) else: sub_block_ = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String( sub_block.namespaced_name), "states": amulet_nbt.TAG_Compound({ key: val for key, val in properties.items() if isinstance(val, PropertyDataTypes) }), "version": amulet_nbt.TAG_Int(sub_block_version), })) full_block.append(sub_block_) palette[index] = tuple(full_block) chunk = [] for cy in range(16): if cy in blocks: palette_index, sub_chunk = fast_unique( blocks.get_sub_chunk(cy)) sub_chunk_palette = palette[palette_index] sub_chunk_depth = palette_depth[palette_index].max() if (sub_chunk_depth == 1 and len(sub_chunk_palette) == 1 and sub_chunk_palette[0][0]["name"].value == "minecraft:air"): chunk.append(None) else: # pad block_palette with air in the extra layers sub_chunk_palette_full = numpy.empty( (sub_chunk_palette.size, sub_chunk_depth), dtype=object) sub_chunk_palette_full.fill(air) for index, block_tuple in enumerate(sub_chunk_palette): for sub_index, block in enumerate(block_tuple): sub_chunk_palette_full[index, sub_index] = block # should now be a 2D array with an amulet_nbt.NBTFile in each element sub_chunk_bytes = [b"\x08", bytes([sub_chunk_depth])] for sub_chunk_layer_index in range(sub_chunk_depth): # TODO: sort out a way to do this quicker without brute forcing it. ( sub_chunk_layer_palette, sub_chunk_remap, ) = brute_sort_objects_no_hash( sub_chunk_palette_full[:, sub_chunk_layer_index]) sub_chunk_layer = sub_chunk_remap[ sub_chunk.ravel()] # sub_chunk_layer, sub_chunk_layer_palette = sub_chunk, sub_chunk_palette_full[:, sub_chunk_layer_index] sub_chunk_bytes.append( self._save_palette_subchunk( sub_chunk_layer.reshape(16, 16, 16), list(sub_chunk_layer_palette.ravel()), )) chunk.append(b"".join(sub_chunk_bytes)) else: chunk.append(None) else: chunk = [None] * 16 return chunk
def _load_subchunks( self, subchunks: List[None, bytes] ) -> Tuple[Dict[int, SubChunkNDArray], AnyNDArray]: """ Load a list of bytes objects which contain chunk data This function should be able to load all sub-chunk formats (technically before it) All sub-chunks will almost certainly all have the same sub-chunk version but it should be able to handle a case where that is not true. As such this function will return a Chunk and a rather complicated block_palette The newer formats allow multiple blocks to occupy the same space and the newer versions also include a version ber block. So this will also need returning for the translator to handle. The block_palette will be a numpy array containing tuple objects The tuple represents the "block" however can contain more than one Block object. Inside the tuple are one or more tuples. These include the block version number and the block itself The block version number will be either None if no block version is given or a tuple containing 4 ints. The block will be either a Block class for the newer formats or a tuple of two ints for the older formats """ blocks: Dict[int, SubChunkNDArray] = {} palette: List[Tuple[Tuple[Optional[int], Union[Tuple[int, int], Block], ], ...]] = [(( 17563649, Block( namespace="minecraft", base_name="air", properties={ "block_data": amulet_nbt.TAG_Int(0) }, ), ), )] for cy, data in enumerate(subchunks): if data is None: continue if data[0] in [0, 2, 3, 4, 5, 6, 7]: block_ids = numpy.frombuffer(data[1:1 + 2**12], dtype=numpy.uint8).astype( numpy.uint16) block_data = from_nibble_array( numpy.frombuffer(data[1 + 2**12:1 + 2**12 + 2**11], dtype=numpy.uint8)) combined_palette, block_array = fast_unique( numpy.transpose( ((block_ids << 4) + block_data).reshape(16, 16, 16), (0, 2, 1))) blocks[cy] = block_array + len(palette) for b in numpy.array( [combined_palette >> 4, combined_palette & 15]).T: palette.append(((None, tuple(b)), )) elif data[0] in [1, 8]: if data[0] == 1: storage_count = 1 data = data[1:] else: storage_count, data = data[1], data[2:] sub_chunk_blocks = numpy.zeros((16, 16, 16, storage_count), dtype=numpy.uint32) sub_chunk_palette: List[List[Tuple[Optional[int], Block]]] = [] for storage_index in range(storage_count): ( sub_chunk_blocks[:, :, :, storage_index], palette_data, data, ) = self._load_palette_blocks(data) palette_data_out: List[Tuple[Optional[int], Block]] = [] for block in palette_data: namespace, base_name = block["name"].value.split( ":", 1) if "version" in block: version: Optional[int] = block["version"].value else: version = None if "states" in block: # 1.13 format properties = block["states"].value if version is None: version = 17694720 # 1, 14, 0, 0 else: properties = { "block_data": amulet_nbt.TAG_Int(block["val"].value) } palette_data_out.append(( version, Block( namespace=namespace, base_name=base_name, properties=properties, ), )) sub_chunk_palette.append(palette_data_out) if storage_count == 1: blocks[cy] = sub_chunk_blocks[:, :, :, 0] + len(palette) palette += [(val, ) for val in sub_chunk_palette[0]] elif storage_count > 1: # we have two or more storages so need to find the unique block combinations and merge them together sub_chunk_palette_, sub_chunk_blocks = numpy.unique( sub_chunk_blocks.reshape(-1, storage_count), return_inverse=True, axis=0, ) blocks[cy] = sub_chunk_blocks.reshape(16, 16, 16).astype( numpy.uint32) + len(palette) palette += [ tuple( sub_chunk_palette[storage_index][index] for storage_index, index in enumerate( palette_indexes) if not (storage_index > 0 and sub_chunk_palette[storage_index][index] [1].namespaced_name == "minecraft:air")) for palette_indexes in sub_chunk_palette_ ] else: continue # block_palette should now look like this # List[ # Tuple[ # Tuple[version, Block], ... # ] # ] numpy_palette, lut = brute_sort_objects(palette) for cy in blocks.keys(): blocks[cy] = lut[blocks[cy]] return blocks, numpy_palette
def _encode_subchunks( self, blocks: "Blocks", palette: AnyNDArray, bounds: Tuple[int, int], max_world_version: VersionNumberTuple, ) -> Dict[int, Optional[bytes]]: # Encode sub-chunk block format 8 palette_depth = numpy.array([len(block) for block in palette]) min_y = bounds[0] // 16 max_y = bounds[1] // 16 if palette.size: if palette[0][0][0] is None: air = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String("minecraft:air"), "val": amulet_nbt.TAG_Short(0), })) else: air = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String("minecraft:air"), "states": amulet_nbt.TAG_Compound({}), "version": amulet_nbt.TAG_Int(17_629_184), # 1, 13, 0, 0 })) for index, block in enumerate(palette): block: Tuple[Tuple[Optional[int], Block], ...] full_block = [] for sub_block_version, sub_block in block: properties = sub_block.properties if sub_block_version is None: block_data = properties.get("block_data", amulet_nbt.TAG_Int(0)) if isinstance(block_data, amulet_nbt.TAG_Int): block_data = block_data.value # if block_data >= 16: # block_data = 0 else: block_data = 0 sub_block_ = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String( sub_block.namespaced_name), "val": amulet_nbt.TAG_Short(block_data), })) else: sub_block_ = amulet_nbt.NBTFile( amulet_nbt.TAG_Compound({ "name": amulet_nbt.TAG_String( sub_block.namespaced_name), "states": amulet_nbt.TAG_Compound({ key: val for key, val in properties.items() if isinstance(val, PropertyDataTypes) }), "version": amulet_nbt.TAG_Int(sub_block_version), })) full_block.append(sub_block_) palette[index] = tuple(full_block) chunk = {} for cy in range(min_y, max_y): if cy in blocks: palette_index, sub_chunk = fast_unique( blocks.get_sub_chunk(cy)) sub_chunk_palette = palette[palette_index] sub_chunk_depth = palette_depth[palette_index].max() if (sub_chunk_depth == 1 and len(sub_chunk_palette) == 1 and sub_chunk_palette[0][0]["name"].value == "minecraft:air"): chunk[cy] = None else: # pad block_palette with air in the extra layers sub_chunk_palette_full = numpy.empty( (sub_chunk_palette.size, sub_chunk_depth), dtype=object) sub_chunk_palette_full.fill(air) for index, block_tuple in enumerate(sub_chunk_palette): for sub_index, block in enumerate(block_tuple): sub_chunk_palette_full[index, sub_index] = block # should now be a 2D array with an amulet_nbt.NBTFile in each element if max_world_version[1] >= ( 1, 17, 30, ): # Why do I need to check against game version and not chunk version sub_chunk_bytes = [ b"\x09", bytes([sub_chunk_depth]), struct.pack("b", cy), ] else: sub_chunk_bytes = [ b"\x08", bytes([sub_chunk_depth]) ] for sub_chunk_layer_index in range(sub_chunk_depth): # TODO: sort out a way to do this quicker without brute forcing it. ( sub_chunk_layer_palette, sub_chunk_remap, ) = brute_sort_objects_no_hash( sub_chunk_palette_full[:, sub_chunk_layer_index]) sub_chunk_layer = sub_chunk_remap[ sub_chunk.ravel()] # sub_chunk_layer, sub_chunk_layer_palette = sub_chunk, sub_chunk_palette_full[:, sub_chunk_layer_index] sub_chunk_bytes.append( self._save_palette_subchunk( sub_chunk_layer.reshape(16, 16, 16), list(sub_chunk_layer_palette.ravel()), )) chunk[cy] = b"".join(sub_chunk_bytes) else: chunk[cy] = None else: chunk = {i: None for i in range(min_y, max_y)} return chunk
def decode(self, cx: int, cz: int, data: Dict[bytes, bytes], bounds: Tuple[int, int]) -> Tuple[Chunk, AnyNDArray]: """ Create an amulet.api.chunk.Chunk object from raw data given by the format :param cx: chunk x coordinate :param cz: chunk z coordinate :param data: Raw chunk data provided by the format. :param bounds: The minimum and maximum height of the chunk. :return: Chunk object in version-specific format, along with the block_palette for that chunk. """ chunk = Chunk(cx, cz) chunk_palette = numpy.empty(0, dtype=object) chunk.misc = {"bedrock_chunk_data": data} data.pop(b"v", None) data.pop(b",", None) if self._features["terrain"].startswith( "2f"): # ["2farray", "2f1palette", "2fnpalette"] subchunks = {} for key in data.copy().keys(): if len(key) == 2 and key[0:1] == b"\x2F": cy = struct.unpack("b", key[1:2])[0] subchunks[self._chunk_key_to_sub_chunk( cy, bounds[0] >> 4)] = data.pop(key) chunk.blocks, chunk_palette = self._load_subchunks(subchunks) elif self._features["terrain"] == "30array": chunk_data = data.pop(b"\x30", None) if chunk_data is not None: block_ids = numpy.frombuffer(chunk_data[:2**15], dtype=numpy.uint8).astype( numpy.uint16) block_data = from_nibble_array( numpy.frombuffer(chunk_data[2**15:2**15 + 2**14], dtype=numpy.uint8)) # there is other data here but we are going to skip over it combined_palette, block_array = fast_unique( numpy.transpose( ((block_ids << 4) + block_data).reshape(16, 16, 128), (0, 2, 1))) chunk.blocks = { i: block_array[:, i * 16:(i + 1) * 16, :] for i in range(8) } palette: AnyNDArray = numpy.array( [combined_palette >> 4, combined_palette & 15]).T chunk_palette = numpy.empty(len(palette), dtype=object) for i, b in enumerate(palette): chunk_palette[i] = ((None, tuple(b)), ) else: raise Exception if self._features["finalised_state"] == "int0-2": state = data.pop(b"\x36", None) val = 2 if isinstance(state, bytes): if len(state) == 1: # old versions of the game store this as a byte val = struct.unpack("b", state)[0] elif len(state) == 4: # newer versions store it as an int val = struct.unpack("<i", state)[0] chunk.status = val if b"+" in data: height, biome = self._decode_height_3d_biomes( data[b"+"], bounds[0] >> 4) chunk.misc["height"] = height chunk.biomes = biome elif b"\x2D" in data: d2d = data[b"\x2D"] height, biome = ( numpy.frombuffer(d2d[:512], "<i2").reshape((16, 16)), d2d[512:], ) chunk.misc["height"] = height chunk.biomes = numpy.frombuffer(biome, dtype="uint8").reshape(16, 16).T # TODO: implement key support # \x2D heightmap and biomes # \x31 block entity # \x32 entity # \x33 ticks # \x34 block extra data # \x35 biome state # \x39 7 ints and an end (03)? Honestly don't know what this is # \x3A fire tick? # \x2E 2d legacy # \x30 legacy terrain # unpack block entities and entities if self._features["block_entities"] == "31list": block_entities = self._unpack_nbt_list(data.pop(b"\x31", b"")) chunk.block_entities = self._decode_block_entity_list( block_entities) if self._features["entities"] == "32list" and amulet.entity_support: entities = self._unpack_nbt_list(data.pop(b"\x32", b"")) chunk.entities = self._decode_entity_list(entities) return chunk, chunk_palette