def _save_subchunks_1(self, blocks: "Blocks",
palette: AnyNDArray) -> List[Optional[bytes]]:
for index, block in enumerate(palette):
block: Tuple[Tuple[None, Block], ...]
block_data = block[0][1].properties.get("block_data",
amulet_nbt.TAG_Int(0))
if isinstance(block_data, amulet_nbt.TAG_Int):
block_data = block_data.value
# if block_data >= 16:
# block_data = 0
else:
block_data = 0
palette[index] = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String(block[0][1].namespaced_name),
"val":
amulet_nbt.TAG_Short(block_data),
}))
chunk = []
for cy in range(16):
if cy in blocks:
palette_index, sub_chunk = fast_unique(
blocks.get_sub_chunk(cy))
sub_chunk_palette = list(palette[palette_index])
chunk.append(b"\x01" + self._save_palette_subchunk(
sub_chunk.ravel(), sub_chunk_palette))
else:
chunk.append(None)
return chunk
def _decode_blocks(
self, chunk_sections: nbt.TAG_List
) -> Tuple[Dict[int, SubChunkNDArray], AnyNDArray]:
blocks: Dict[int, SubChunkNDArray] = {}
palette = []
palette_len = 0
for section in chunk_sections:
cy: int = section["Y"].value
section_blocks = numpy.frombuffer(
section["Blocks"].value, dtype=numpy.uint8
)
del section["Blocks"]
section_data = numpy.frombuffer(section["Data"].value, dtype=numpy.uint8)
del section["Data"]
section_blocks = section_blocks.reshape((16, 16, 16))
section_blocks = section_blocks.astype(numpy.uint16)
section_data = world_utils.from_nibble_array(section_data)
section_data = section_data.reshape((16, 16, 16))
if "Add" in section:
add_blocks = numpy.frombuffer(section["Add"].value, dtype=numpy.uint8)
del section["Add"]
add_blocks = world_utils.from_nibble_array(add_blocks)
add_blocks = add_blocks.reshape((16, 16, 16))
section_blocks |= add_blocks.astype(numpy.uint16) << 8
# TODO: fix this
(section_palette, blocks[cy]) = world_utils.fast_unique(
numpy.transpose(
(section_blocks << 4) + section_data, (2, 0, 1)
) # YZX -> XYZ
)
blocks[cy] += palette_len
palette_len += len(section_palette)
palette.append(section_palette)
if palette:
final_palette, lut = numpy.unique(
numpy.concatenate(palette), return_inverse=True
)
final_palette: numpy.ndarray = numpy.array(
[final_palette >> 4, final_palette & 15]
).T
for cy in blocks:
blocks[cy] = lut[blocks[cy]]
else:
final_palette = numpy.array([], dtype=numpy.object)
return blocks, final_palette
def _decode_blocks(
self, chunk: Chunk, chunk_sections: Dict[int, TAG_Compound]
) -> AnyNDArray:
blocks: Dict[int, SubChunkNDArray] = {}
palette = []
palette_len = 0
for cy, section in chunk_sections.items():
section_blocks = numpy.frombuffer(
section.pop("Blocks").value, dtype=numpy.uint8
)
section_data = numpy.frombuffer(
section.pop("Data").value, dtype=numpy.uint8
)
section_blocks = section_blocks.reshape((16, 16, 16))
section_blocks = section_blocks.astype(numpy.uint16)
section_data = world_utils.from_nibble_array(section_data)
section_data = section_data.reshape((16, 16, 16))
if "Add" in section:
add_blocks = numpy.frombuffer(
section.pop("Add").value, dtype=numpy.uint8
)
add_blocks = world_utils.from_nibble_array(add_blocks)
add_blocks = add_blocks.reshape((16, 16, 16))
section_blocks |= add_blocks.astype(numpy.uint16) << 8
# TODO: fix this
(section_palette, blocks[cy]) = world_utils.fast_unique(
numpy.transpose(
(section_blocks << 4) + section_data, (2, 0, 1)
) # YZX -> XYZ
)
blocks[cy] += palette_len
palette_len += len(section_palette)
palette.append(section_palette)
if palette:
final_palette, lut = numpy.unique(
numpy.concatenate(palette), return_inverse=True
)
final_palette: numpy.ndarray = numpy.array(
[final_palette >> 4, final_palette & 15]
).T
for cy in blocks:
blocks[cy] = lut[blocks[cy]]
else:
final_palette = numpy.array([], dtype=object)
chunk.blocks = blocks
return final_palette
def decode(self, cx: int, cz: int,
data: Dict[bytes, bytes]) -> Tuple[Chunk, AnyNDArray]:
# chunk_key_base = struct.pack("<ii", cx, cz)
chunk = Chunk(cx, cz)
chunk_palette = numpy.empty(0, dtype=object)
if self.features["terrain"].startswith(
"2f"): # ["2farray", "2f1palette", "2fnpalette"]
subchunks = [
data.get(b"\x2F" + bytes([i]), None) for i in range(16)
]
chunk.blocks, chunk_palette = self._load_subchunks(subchunks)
elif self.features["terrain"] == "30array":
chunk_data = data.get(b"\x30", None)
if chunk_data is not None:
block_ids = numpy.frombuffer(chunk_data[:2**15],
dtype=numpy.uint8).astype(
numpy.uint16)
block_data = from_nibble_array(
numpy.frombuffer(chunk_data[2**15:2**15 + 2**14],
dtype=numpy.uint8))
# there is other data here but we are going to skip over it
combined_palette, block_array = fast_unique(
numpy.transpose(
((block_ids << 4) + block_data).reshape(16, 16, 128),
(0, 2, 1)))
chunk.blocks = {
i: block_array[:, i * 16:(i + 1) * 16, :]
for i in range(8)
}
palette: AnyNDArray = numpy.array(
[combined_palette >> 4, combined_palette & 15]).T
chunk_palette = numpy.empty(len(palette), dtype=object)
for i, b in enumerate(palette):
chunk_palette[i] = ((None, tuple(b)), )
else:
raise Exception
if self.features["finalised_state"] == "int0-2":
if b"\x36" in data:
val = struct.unpack("<i", data[b"\x36"])[0]
else:
val = 2
chunk.status = val
if self.features["data_2d"] in [
"height512|biome256",
"unused_height512|biome256",
]:
d2d = data.get(b"\x2D", b"\x00" * 768)
height, biome = d2d[:512], d2d[512:]
if self.features["data_2d"] == "height512|biome256":
pass # TODO: put this data somewhere
chunk.biomes = numpy.frombuffer(biome,
dtype="uint8").reshape(16, 16)
# TODO: impliment key support
# \x2D heightmap and biomes
# \x31 block entity
# \x32 entity
# \x33 ticks
# \x34 block extra data
# \x35 biome state
# \x39 7 ints and an end (03)? Honestly don't know what this is
# \x3A fire tick?
# \x2E 2d legacy
# \x30 legacy terrain
# unpack block entities and entities
if self.features["block_entities"] == "31list":
block_entities = self._unpack_nbt_list(data.get(b"\x31", b""))
chunk.block_entities = self._decode_block_entities(block_entities)
if self.features["entities"] == "32list" and amulet.entity_support:
entities = self._unpack_nbt_list(data.get(b"\x32", b""))
chunk.entities = self._decode_entities(entities)
return chunk, chunk_palette
def _save_subchunks_8(self, blocks: "Blocks",
palette: AnyNDArray) -> List[Optional[bytes]]:
palette_depth = numpy.array([len(block) for block in palette])
if palette.size:
if palette[0][0][0] is None:
air = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String("minecraft:air"),
"val":
amulet_nbt.TAG_Short(0),
}))
else:
air = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String("minecraft:air"),
"states":
amulet_nbt.TAG_Compound({}),
"version":
amulet_nbt.TAG_Int(17_629_184), # 1, 13, 0, 0
}))
for index, block in enumerate(palette):
block: Tuple[Tuple[Optional[int], Block], ...]
full_block = []
for sub_block_version, sub_block in block:
properties = sub_block.properties
if sub_block_version is None:
block_data = properties.get("block_data",
amulet_nbt.TAG_Int(0))
if isinstance(block_data, amulet_nbt.TAG_Int):
block_data = block_data.value
# if block_data >= 16:
# block_data = 0
else:
block_data = 0
sub_block_ = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String(
sub_block.namespaced_name),
"val":
amulet_nbt.TAG_Short(block_data),
}))
else:
sub_block_ = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String(
sub_block.namespaced_name),
"states":
amulet_nbt.TAG_Compound({
key: val
for key, val in properties.items()
if isinstance(val, PropertyDataTypes)
}),
"version":
amulet_nbt.TAG_Int(sub_block_version),
}))
full_block.append(sub_block_)
palette[index] = tuple(full_block)
chunk = []
for cy in range(16):
if cy in blocks:
palette_index, sub_chunk = fast_unique(
blocks.get_sub_chunk(cy))
sub_chunk_palette = palette[palette_index]
sub_chunk_depth = palette_depth[palette_index].max()
if (sub_chunk_depth == 1 and len(sub_chunk_palette) == 1
and sub_chunk_palette[0][0]["name"].value
== "minecraft:air"):
chunk.append(None)
else:
# pad block_palette with air in the extra layers
sub_chunk_palette_full = numpy.empty(
(sub_chunk_palette.size, sub_chunk_depth),
dtype=object)
sub_chunk_palette_full.fill(air)
for index, block_tuple in enumerate(sub_chunk_palette):
for sub_index, block in enumerate(block_tuple):
sub_chunk_palette_full[index,
sub_index] = block
# should now be a 2D array with an amulet_nbt.NBTFile in each element
sub_chunk_bytes = [b"\x08", bytes([sub_chunk_depth])]
for sub_chunk_layer_index in range(sub_chunk_depth):
# TODO: sort out a way to do this quicker without brute forcing it.
(
sub_chunk_layer_palette,
sub_chunk_remap,
) = brute_sort_objects_no_hash(
sub_chunk_palette_full[:,
sub_chunk_layer_index])
sub_chunk_layer = sub_chunk_remap[
sub_chunk.ravel()]
# sub_chunk_layer, sub_chunk_layer_palette = sub_chunk, sub_chunk_palette_full[:, sub_chunk_layer_index]
sub_chunk_bytes.append(
self._save_palette_subchunk(
sub_chunk_layer.reshape(16, 16, 16),
list(sub_chunk_layer_palette.ravel()),
))
chunk.append(b"".join(sub_chunk_bytes))
else:
chunk.append(None)
else:
chunk = [None] * 16
return chunk
def _load_subchunks(
self, subchunks: List[None, bytes]
) -> Tuple[Dict[int, SubChunkNDArray], AnyNDArray]:
"""
Load a list of bytes objects which contain chunk data
This function should be able to load all sub-chunk formats (technically before it)
All sub-chunks will almost certainly all have the same sub-chunk version but
it should be able to handle a case where that is not true.
As such this function will return a Chunk and a rather complicated block_palette
The newer formats allow multiple blocks to occupy the same space and the
newer versions also include a version ber block. So this will also need
returning for the translator to handle.
The block_palette will be a numpy array containing tuple objects
The tuple represents the "block" however can contain more than one Block object.
Inside the tuple are one or more tuples.
These include the block version number and the block itself
The block version number will be either None if no block version is given
or a tuple containing 4 ints.
The block will be either a Block class for the newer formats or a tuple of two ints for the older formats
"""
blocks: Dict[int, SubChunkNDArray] = {}
palette: List[Tuple[Tuple[Optional[int],
Union[Tuple[int, int], Block], ], ...]] = [((
17563649,
Block(
namespace="minecraft",
base_name="air",
properties={
"block_data":
amulet_nbt.TAG_Int(0)
},
),
), )]
for cy, data in enumerate(subchunks):
if data is None:
continue
if data[0] in [0, 2, 3, 4, 5, 6, 7]:
block_ids = numpy.frombuffer(data[1:1 + 2**12],
dtype=numpy.uint8).astype(
numpy.uint16)
block_data = from_nibble_array(
numpy.frombuffer(data[1 + 2**12:1 + 2**12 + 2**11],
dtype=numpy.uint8))
combined_palette, block_array = fast_unique(
numpy.transpose(
((block_ids << 4) + block_data).reshape(16, 16, 16),
(0, 2, 1)))
blocks[cy] = block_array + len(palette)
for b in numpy.array(
[combined_palette >> 4, combined_palette & 15]).T:
palette.append(((None, tuple(b)), ))
elif data[0] in [1, 8]:
if data[0] == 1:
storage_count = 1
data = data[1:]
else:
storage_count, data = data[1], data[2:]
sub_chunk_blocks = numpy.zeros((16, 16, 16, storage_count),
dtype=numpy.uint32)
sub_chunk_palette: List[List[Tuple[Optional[int], Block]]] = []
for storage_index in range(storage_count):
(
sub_chunk_blocks[:, :, :, storage_index],
palette_data,
data,
) = self._load_palette_blocks(data)
palette_data_out: List[Tuple[Optional[int], Block]] = []
for block in palette_data:
namespace, base_name = block["name"].value.split(
":", 1)
if "version" in block:
version: Optional[int] = block["version"].value
else:
version = None
if "states" in block: # 1.13 format
properties = block["states"].value
if version is None:
version = 17694720 # 1, 14, 0, 0
else:
properties = {
"block_data":
amulet_nbt.TAG_Int(block["val"].value)
}
palette_data_out.append((
version,
Block(
namespace=namespace,
base_name=base_name,
properties=properties,
),
))
sub_chunk_palette.append(palette_data_out)
if storage_count == 1:
blocks[cy] = sub_chunk_blocks[:, :, :, 0] + len(palette)
palette += [(val, ) for val in sub_chunk_palette[0]]
elif storage_count > 1:
# we have two or more storages so need to find the unique block combinations and merge them together
sub_chunk_palette_, sub_chunk_blocks = numpy.unique(
sub_chunk_blocks.reshape(-1, storage_count),
return_inverse=True,
axis=0,
)
blocks[cy] = sub_chunk_blocks.reshape(16, 16, 16).astype(
numpy.uint32) + len(palette)
palette += [
tuple(
sub_chunk_palette[storage_index][index]
for storage_index, index in enumerate(
palette_indexes)
if not (storage_index > 0
and sub_chunk_palette[storage_index][index]
[1].namespaced_name == "minecraft:air"))
for palette_indexes in sub_chunk_palette_
]
else:
continue
# block_palette should now look like this
# List[
# Tuple[
# Tuple[version, Block], ...
# ]
# ]
numpy_palette, lut = brute_sort_objects(palette)
for cy in blocks.keys():
blocks[cy] = lut[blocks[cy]]
return blocks, numpy_palette
def _encode_subchunks(
self,
blocks: "Blocks",
palette: AnyNDArray,
bounds: Tuple[int, int],
max_world_version: VersionNumberTuple,
) -> Dict[int, Optional[bytes]]:
# Encode sub-chunk block format 8
palette_depth = numpy.array([len(block) for block in palette])
min_y = bounds[0] // 16
max_y = bounds[1] // 16
if palette.size:
if palette[0][0][0] is None:
air = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String("minecraft:air"),
"val":
amulet_nbt.TAG_Short(0),
}))
else:
air = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String("minecraft:air"),
"states":
amulet_nbt.TAG_Compound({}),
"version":
amulet_nbt.TAG_Int(17_629_184), # 1, 13, 0, 0
}))
for index, block in enumerate(palette):
block: Tuple[Tuple[Optional[int], Block], ...]
full_block = []
for sub_block_version, sub_block in block:
properties = sub_block.properties
if sub_block_version is None:
block_data = properties.get("block_data",
amulet_nbt.TAG_Int(0))
if isinstance(block_data, amulet_nbt.TAG_Int):
block_data = block_data.value
# if block_data >= 16:
# block_data = 0
else:
block_data = 0
sub_block_ = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String(
sub_block.namespaced_name),
"val":
amulet_nbt.TAG_Short(block_data),
}))
else:
sub_block_ = amulet_nbt.NBTFile(
amulet_nbt.TAG_Compound({
"name":
amulet_nbt.TAG_String(
sub_block.namespaced_name),
"states":
amulet_nbt.TAG_Compound({
key: val
for key, val in properties.items()
if isinstance(val, PropertyDataTypes)
}),
"version":
amulet_nbt.TAG_Int(sub_block_version),
}))
full_block.append(sub_block_)
palette[index] = tuple(full_block)
chunk = {}
for cy in range(min_y, max_y):
if cy in blocks:
palette_index, sub_chunk = fast_unique(
blocks.get_sub_chunk(cy))
sub_chunk_palette = palette[palette_index]
sub_chunk_depth = palette_depth[palette_index].max()
if (sub_chunk_depth == 1 and len(sub_chunk_palette) == 1
and sub_chunk_palette[0][0]["name"].value
== "minecraft:air"):
chunk[cy] = None
else:
# pad block_palette with air in the extra layers
sub_chunk_palette_full = numpy.empty(
(sub_chunk_palette.size, sub_chunk_depth),
dtype=object)
sub_chunk_palette_full.fill(air)
for index, block_tuple in enumerate(sub_chunk_palette):
for sub_index, block in enumerate(block_tuple):
sub_chunk_palette_full[index,
sub_index] = block
# should now be a 2D array with an amulet_nbt.NBTFile in each element
if max_world_version[1] >= (
1,
17,
30,
): # Why do I need to check against game version and not chunk version
sub_chunk_bytes = [
b"\x09",
bytes([sub_chunk_depth]),
struct.pack("b", cy),
]
else:
sub_chunk_bytes = [
b"\x08", bytes([sub_chunk_depth])
]
for sub_chunk_layer_index in range(sub_chunk_depth):
# TODO: sort out a way to do this quicker without brute forcing it.
(
sub_chunk_layer_palette,
sub_chunk_remap,
) = brute_sort_objects_no_hash(
sub_chunk_palette_full[:,
sub_chunk_layer_index])
sub_chunk_layer = sub_chunk_remap[
sub_chunk.ravel()]
# sub_chunk_layer, sub_chunk_layer_palette = sub_chunk, sub_chunk_palette_full[:, sub_chunk_layer_index]
sub_chunk_bytes.append(
self._save_palette_subchunk(
sub_chunk_layer.reshape(16, 16, 16),
list(sub_chunk_layer_palette.ravel()),
))
chunk[cy] = b"".join(sub_chunk_bytes)
else:
chunk[cy] = None
else:
chunk = {i: None for i in range(min_y, max_y)}
return chunk
def decode(self, cx: int, cz: int, data: Dict[bytes, bytes],
bounds: Tuple[int, int]) -> Tuple[Chunk, AnyNDArray]:
"""
Create an amulet.api.chunk.Chunk object from raw data given by the format
:param cx: chunk x coordinate
:param cz: chunk z coordinate
:param data: Raw chunk data provided by the format.
:param bounds: The minimum and maximum height of the chunk.
:return: Chunk object in version-specific format, along with the block_palette for that chunk.
"""
chunk = Chunk(cx, cz)
chunk_palette = numpy.empty(0, dtype=object)
chunk.misc = {"bedrock_chunk_data": data}
data.pop(b"v", None)
data.pop(b",", None)
if self._features["terrain"].startswith(
"2f"): # ["2farray", "2f1palette", "2fnpalette"]
subchunks = {}
for key in data.copy().keys():
if len(key) == 2 and key[0:1] == b"\x2F":
cy = struct.unpack("b", key[1:2])[0]
subchunks[self._chunk_key_to_sub_chunk(
cy, bounds[0] >> 4)] = data.pop(key)
chunk.blocks, chunk_palette = self._load_subchunks(subchunks)
elif self._features["terrain"] == "30array":
chunk_data = data.pop(b"\x30", None)
if chunk_data is not None:
block_ids = numpy.frombuffer(chunk_data[:2**15],
dtype=numpy.uint8).astype(
numpy.uint16)
block_data = from_nibble_array(
numpy.frombuffer(chunk_data[2**15:2**15 + 2**14],
dtype=numpy.uint8))
# there is other data here but we are going to skip over it
combined_palette, block_array = fast_unique(
numpy.transpose(
((block_ids << 4) + block_data).reshape(16, 16, 128),
(0, 2, 1)))
chunk.blocks = {
i: block_array[:, i * 16:(i + 1) * 16, :]
for i in range(8)
}
palette: AnyNDArray = numpy.array(
[combined_palette >> 4, combined_palette & 15]).T
chunk_palette = numpy.empty(len(palette), dtype=object)
for i, b in enumerate(palette):
chunk_palette[i] = ((None, tuple(b)), )
else:
raise Exception
if self._features["finalised_state"] == "int0-2":
state = data.pop(b"\x36", None)
val = 2
if isinstance(state, bytes):
if len(state) == 1:
# old versions of the game store this as a byte
val = struct.unpack("b", state)[0]
elif len(state) == 4:
# newer versions store it as an int
val = struct.unpack("<i", state)[0]
chunk.status = val
if b"+" in data:
height, biome = self._decode_height_3d_biomes(
data[b"+"], bounds[0] >> 4)
chunk.misc["height"] = height
chunk.biomes = biome
elif b"\x2D" in data:
d2d = data[b"\x2D"]
height, biome = (
numpy.frombuffer(d2d[:512], "<i2").reshape((16, 16)),
d2d[512:],
)
chunk.misc["height"] = height
chunk.biomes = numpy.frombuffer(biome,
dtype="uint8").reshape(16, 16).T
# TODO: implement key support
# \x2D heightmap and biomes
# \x31 block entity
# \x32 entity
# \x33 ticks
# \x34 block extra data
# \x35 biome state
# \x39 7 ints and an end (03)? Honestly don't know what this is
# \x3A fire tick?
# \x2E 2d legacy
# \x30 legacy terrain
# unpack block entities and entities
if self._features["block_entities"] == "31list":
block_entities = self._unpack_nbt_list(data.pop(b"\x31", b""))
chunk.block_entities = self._decode_block_entity_list(
block_entities)
if self._features["entities"] == "32list" and amulet.entity_support:
entities = self._unpack_nbt_list(data.pop(b"\x32", b""))
chunk.entities = self._decode_entity_list(entities)
return chunk, chunk_palette
