def write_block(self, level, first_record, last_record, payload, zpayload):
if not (0 <= level < FIRST_EXTENSION_LEVEL):
raise ZSError("invalid level %s" % (level,))
if level == 0:
self._hasher.update(payload)
block_offset = self._file.tell()
block_contents = six.int2byte(level) + zpayload
write_uleb128(len(block_contents), self._file)
self._file.write(block_contents)
self._file.write(encoded_crc64xz(block_contents))
total_block_length = self._file.tell() - block_offset
self._spin(total_block_length, 1, False)
if level >= len(self._level_entries):
# First block we've seen at this level
assert level == len(self._level_entries)
self._level_entries.append([])
# This can only happen if all the previous levels just flushed.
for i in range(level):
assert not self._level_entries[i]
entries = self._level_entries[level]
entries.append((first_record, last_record,
block_offset, total_block_length))
if len(entries) >= self._branching_factor:
self._flush_index(level)
def finish(self):
"""Declare this file finished.
This method writes out the root block, updates the header, etc.
Importantly, we do not write out the correct magic number until this
method completes, so no ZS reader will be willing to read your file
until this is called (see :ref:`magic-numbers`).
Do not call this method unless you are sure you have added the right
records. (In particular, you definitely don't want to call this from a
``finally`` block, or automatically from a ``with`` block context
manager.)
Calls :meth:`close`.
"""
self._check_open()
with errors_close(self):
# Stop all the processing queues and wait for them to finish.
for i in range(self._parallelism):
#sys.stderr.write("putting QUIT\n"); sys.stderr.flush()
self._safe_put(self._compress_queue, _QUIT)
for compressor in self._compressors:
self._safe_join(compressor)
#sys.stdout.write("All compressors finished; waiting for writer\n")
# All compressors have now finished their work, and submitted
# everything to the write queue.
self._safe_put(self._write_queue, _QUIT)
self._safe_join(self._writer)
# The writer and compressors have all exited, so any errors they've
# encountered have definitely been enqueued.
self._check_error()
sys.stdout.write("zs: Updating header...\n")
root_index_offset, root_index_length, sha256 = self._finish_queue.get()
#sys.stdout.write("zs: Root index offset: %s\n" % (root_index_offset,))
# Now we have the root offset
self._header["root_index_offset"] = root_index_offset
self._header["root_index_length"] = root_index_length
self._header["sha256"] = sha256
# And can get the total file length
self._file.seek(0, 2)
self._header["total_file_length"] = self._file.tell()
new_encoded_header = _encode_header(self._header)
self._file.seek(len(MAGIC))
# Read the header length and make sure it hasn't changed
old_length, = read_format(self._file, header_data_length_format)
if old_length != len(new_encoded_header):
raise ZSError("header data length changed")
self._file.write(new_encoded_header)
self._file.write(encoded_crc64xz(new_encoded_header))
# Flush the file to disk to make sure that all data is consistent
# before we mark the file as complete.
_flush_file(self._file)
# And now we can write the MAGIC value to mark the file as complete.
self._file.seek(0)
self._file.write(MAGIC)
_flush_file(self._file)
# Done!
self.close()
