class ContainerWriter(object): def __init__(self, fp, schema, sync_marker=None): self.writer = Writer(schema) self.fp = fp self.sync_marker = sync_marker or os.urandom(16) self.header_written = sync_marker is not None self.records = 0 self.buffer = BytesIO() def write_header(self): assert not self.header_written, "Header is already written once" Writer(HEADER_SCHEMA).write(self.fp, { "magic": b"Obj\x01", "meta": { "avro.schema": json.dumps(self.schema.json).encode("utf8"), "avro.codec": b"null" }, "sync": self.sync_marker }) self.header_written = True def write(self, message): self.writer.write(self.buffer, message) self.records += 1 if self.buffer.tell() > 1024 ** 2: self.flush() def flush(self): if not self.header_written: self.write_header() self.header_written = True if not self.records: return write_long(self.fp, self.records) write_long(self.fp, self.buffer.tell()) self.fp.write(self.buffer.getbuffer()) self.fp.write(self.sync_marker) self.fp.flush() self.records = 0 self.buffer = BytesIO() @property def schema(self): """Returns the :class:`avrolight.schema.Schema` instance that this writer uses.""" return self.writer.schema def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.flush() close = flush