def readChunkCompressed(self, cx, cz):
"""
Read a chunk and return its compression type and the compressed data as a (data, fmt) tuple
"""
cx &= 0x1f
cz &= 0x1f
offset = self._getOffset(cx, cz)
if offset == 0:
raise ChunkNotPresent((cx, cz))
sectorStart = offset >> 8
numSectors = offset & 0xff
if numSectors == 0:
raise ChunkNotPresent((cx, cz))
if sectorStart + numSectors > len(self.freeSectors):
raise ChunkNotPresent((cx, cz))
with file(self.path, "rb") as f:
f.seek(sectorStart * self.SECTOR_BYTES)
data = f.read(numSectors * self.SECTOR_BYTES)
if len(data) < 5:
raise RegionFormatError(
"Chunk %s data is only %d bytes long (expected 5)" %
((cx, cz), len(data)))
# region_debug("REGION LOAD {0},{1} sector {2}".format(cx, cz, sectorStart))
length = struct.unpack_from(">I", data)[0]
fmt = struct.unpack_from("B", data, 4)[0]
data = data[5:length + 5]
return data, fmt
def readChunkBytes(self, cx, cz):
"""
:param cx:
:type cx:
:param cz:
:type cz:
:return:
:rtype: bytes
"""
data, fmt = self.readChunkCompressed(cx, cz)
if data is None:
return None
if fmt == self.VERSION_GZIP:
return nbt.gunzip(data)
if fmt == self.VERSION_DEFLATE:
return inflate(data)
raise RegionFormatError("Unknown compress format: {0}".format(fmt))
def writeChunkCompressed(self, cx, cz, data, format):
cx &= 0x1f
cz &= 0x1f
offset = self._getOffset(cx, cz)
sectorNumber = offset >> 8
sectorsAllocated = offset & 0xff
sectorsNeeded = (len(data) +
self.CHUNK_HEADER_SIZE) / self.SECTOR_BYTES + 1
if sectorsNeeded >= 256:
err = RegionFormatError(
"Cannot save chunk %s with compressed length %s (exceeds 1 megabyte)"
% ((cx, cz), len(data)))
err.chunkPosition = cx, cz
if sectorNumber != 0 and sectorsAllocated >= sectorsNeeded:
region_debug("REGION SAVE {0},{1} rewriting {2}b".format(
cx, cz, len(data)))
self.writeSector(sectorNumber, data, format)
else:
# we need to allocate new sectors
# mark the sectors previously used for this chunk as free
for i in xrange(sectorNumber, sectorNumber + sectorsAllocated):
self.freeSectors[i] = True
runLength = 0
runStart = 0
try:
runStart = self.freeSectors.index(True)
for i in range(runStart, len(self.freeSectors)):
if runLength:
if self.freeSectors[i]:
runLength += 1
else:
runLength = 0
elif self.freeSectors[i]:
runStart = i
runLength = 1
if runLength >= sectorsNeeded:
break
except ValueError:
pass
# we found a free space large enough
if runLength >= sectorsNeeded:
region_debug("REGION SAVE {0},{1}, reusing {2}b".format(
cx, cz, len(data)))
sectorNumber = runStart
self._setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
self.writeSector(sectorNumber, data, format)
self.freeSectors[sectorNumber:sectorNumber +
sectorsNeeded] = [False] * sectorsNeeded
else:
# no free space large enough found -- we need to grow the
# file
region_debug("REGION SAVE {0},{1}, growing by {2}b".format(
cx, cz, len(data)))
with file(self.path, "rb+") as f:
f.seek(0, 2)
filesize = f.tell()
sectorNumber = len(self.freeSectors)
assert sectorNumber * self.SECTOR_BYTES == filesize
filesize += sectorsNeeded * self.SECTOR_BYTES
f.truncate(filesize)
self.freeSectors += [False] * sectorsNeeded
self._setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
self.writeSector(sectorNumber, data, format)
self.setTimestamp(cx, cz)
def repair(self):
"""
Fix the following problems with the region file:
- remove offset table entries pointing past the end of the file
- remove entries that overlap other entries
- relocate offsets for chunks whose xPos,yPos don't match
"""
lostAndFound = {}
_freeSectors = [True] * len(self.freeSectors)
_freeSectors[0] = _freeSectors[1] = False
deleted = 0
recovered = 0
log.info("Beginning repairs on {file} ({chunks} chunks)".format(
file=os.path.basename(self.path), chunks=sum(self.offsets > 0)))
for index, offset in enumerate(self.offsets):
if offset:
cx = index & 0x1f
cz = index >> 5
sectorStart = offset >> 8
sectorCount = offset & 0xff
try:
if sectorStart + sectorCount > len(self.freeSectors):
raise RegionFormatError(
"Offset {start}:{end} ({offset}) at index {index} pointed outside of "
"the file".format(start=sectorStart,
end=sectorStart + sectorCount,
index=index,
offset=offset))
data = self.readChunkBytes(cx, cz)
chunkTag = nbt.load(buf=data)
lev = chunkTag["Level"]
xPos = lev["xPos"].value & 0x1f
zPos = lev["zPos"].value & 0x1f
overlaps = False
for i in xrange(sectorStart, sectorStart + sectorCount):
if _freeSectors[i] is False:
overlaps = True
_freeSectors[i] = False
if xPos != cx or zPos != cz:
lostAndFound[xPos, zPos] = data
raise RegionFormatError(
"Chunk {found} was found in the slot reserved for {expected}"
.format(found=(xPos, zPos), expected=(cx, cz)))
if overlaps:
raise RegionFormatError(
"Chunk {found} (in slot {expected}) has overlapping sectors with another chunk!"
.format(found=(xPos, zPos), expected=(cx, cz)))
except Exception as e:
log.info(
"Unexpected chunk data at sector {sector} ({exc})".
format(sector=sectorStart, exc=e))
self._setOffset(cx, cz, 0)
deleted += 1
for cPos, foundData in lostAndFound.iteritems():
cx, cz = cPos
if self._getOffset(cx, cz) == 0:
log.info(
"Found chunk {found} and its slot is empty, recovering it".
format(found=cPos))
self.writeChunkBytes(cx, cz, foundData)
recovered += 1
log.info(
"Repair complete. Removed {0} chunks, recovered {1} chunks, net {2}"
.format(deleted, recovered, recovered - deleted))
def writeChunkCompressed(self, cx, cz, data, format):
cx &= 0x1f
cz &= 0x1f
offset = self._getOffset(cx, cz)
sectorNumber = offset >> 8
sectorsAllocated = offset & 0xff
sectorsNeeded = (len(data) + self.CHUNK_HEADER_SIZE) / self.SECTOR_BYTES + 1
if sectorsNeeded >= 256:
err = RegionFormatError("Cannot save chunk %s with compressed length %s (exceeds 1 megabyte)" %
((cx, cz), len(data)))
err.chunkPosition = cx, cz
if sectorNumber != 0 and sectorsAllocated >= sectorsNeeded:
log.debug("REGION SAVE {0},{1} rewriting {2}b".format(cx, cz, len(data)))
self.writeSector(sectorNumber, data, format)
else:
# we need to allocate new sectors
# mark the sectors previously used for this chunk as free
for i in xrange(sectorNumber, sectorNumber + sectorsAllocated):
self.freeSectors[i] = True
runLength = 0
runStart = 0
try:
runStart = self.freeSectors.index(True)
for i in range(runStart, len(self.freeSectors)):
if runLength:
if self.freeSectors[i]:
runLength += 1
else:
runLength = 0
elif self.freeSectors[i]:
runStart = i
runLength = 1
if runLength >= sectorsNeeded:
break
except ValueError:
pass
# we found a free space large enough
if runLength >= sectorsNeeded:
log.debug("REGION SAVE {0},{1}, reusing {2}b".format(cx, cz, len(data)))
sectorNumber = runStart
self._setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
self.writeSector(sectorNumber, data, format)
self.freeSectors[sectorNumber:sectorNumber + sectorsNeeded] = [False] * sectorsNeeded
else:
# no free space large enough found -- we need to grow the
# file
log.debug("REGION SAVE {0},{1}, growing by {2}b".format(cx, cz, len(data)))
with file(self.path, "rb+") as f:
f.seek(0, 2)
filesize = f.tell()
sectorNumber = len(self.freeSectors)
assert sectorNumber * self.SECTOR_BYTES == filesize
filesize += sectorsNeeded * self.SECTOR_BYTES
f.truncate(filesize)
self.freeSectors += [False] * sectorsNeeded
self._setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
self.writeSector(sectorNumber, data, format)
self.setTimestamp(cx, cz)