def write_block(self, level, first_record, last_record, payload, zpayload): if not (0 <= level < FIRST_EXTENSION_LEVEL): raise ZSError("invalid level %s" % (level,)) if level == 0: self._hasher.update(payload) block_offset = self._file.tell() block_contents = six.int2byte(level) + zpayload write_uleb128(len(block_contents), self._file) self._file.write(block_contents) self._file.write(encoded_crc64xz(block_contents)) total_block_length = self._file.tell() - block_offset self._spin(total_block_length, 1, False) if level >= len(self._level_entries): # First block we've seen at this level assert level == len(self._level_entries) self._level_entries.append([]) # This can only happen if all the previous levels just flushed. for i in range(level): assert not self._level_entries[i] entries = self._level_entries[level] entries.append((first_record, last_record, block_offset, total_block_length)) if len(entries) >= self._branching_factor: self._flush_index(level)
def finish(self): """Declare this file finished. This method writes out the root block, updates the header, etc. Importantly, we do not write out the correct magic number until this method completes, so no ZS reader will be willing to read your file until this is called (see :ref:`magic-numbers`). Do not call this method unless you are sure you have added the right records. (In particular, you definitely don't want to call this from a ``finally`` block, or automatically from a ``with`` block context manager.) Calls :meth:`close`. """ self._check_open() with errors_close(self): # Stop all the processing queues and wait for them to finish. for i in range(self._parallelism): #sys.stderr.write("putting QUIT\n"); sys.stderr.flush() self._safe_put(self._compress_queue, _QUIT) for compressor in self._compressors: self._safe_join(compressor) #sys.stdout.write("All compressors finished; waiting for writer\n") # All compressors have now finished their work, and submitted # everything to the write queue. self._safe_put(self._write_queue, _QUIT) self._safe_join(self._writer) # The writer and compressors have all exited, so any errors they've # encountered have definitely been enqueued. self._check_error() sys.stdout.write("zs: Updating header...\n") root_index_offset, root_index_length, sha256 = self._finish_queue.get() #sys.stdout.write("zs: Root index offset: %s\n" % (root_index_offset,)) # Now we have the root offset self._header["root_index_offset"] = root_index_offset self._header["root_index_length"] = root_index_length self._header["sha256"] = sha256 # And can get the total file length self._file.seek(0, 2) self._header["total_file_length"] = self._file.tell() new_encoded_header = _encode_header(self._header) self._file.seek(len(MAGIC)) # Read the header length and make sure it hasn't changed old_length, = read_format(self._file, header_data_length_format) if old_length != len(new_encoded_header): raise ZSError("header data length changed") self._file.write(new_encoded_header) self._file.write(encoded_crc64xz(new_encoded_header)) # Flush the file to disk to make sure that all data is consistent # before we mark the file as complete. _flush_file(self._file) # And now we can write the MAGIC value to mark the file as complete. self._file.seek(0) self._file.write(MAGIC) _flush_file(self._file) # Done! self.close()