def flush(self):
"""Flush pool contents."""
buf = cStringIO.StringIO()
with records.RecordsWriter(buf) as w:
for record in self._buffer:
w.write(record)
w._pad_block()
str_buf = buf.getvalue()
buf.close()
if not self._exclusive and len(str_buf) > _FILES_API_MAX_SIZE:
raise errors.Error(
"Buffer too big. Can't write more than %s bytes in one request: "
"risk of writes interleaving. Got: %s" %
(_FILES_API_MAX_SIZE, len(str_buf)))
start_time = time.time()
with files.open(self._filename, "a",
exclusive_lock=self._exclusive) as f:
f.write(str_buf)
if self._ctx:
operation.counters.Increment(COUNTER_IO_WRITE_BYTES,
len(str_buf))(self._ctx)
if self._ctx:
operation.counters.Increment(
COUNTER_IO_WRITE_MSEC, int(
(time.time() - start_time) * 1000))(self._ctx)
self._buffer = []
self._size = 0
gc.collect()
def __init__(self, streaming_buffer, writer_spec=None):
"""Initialize a CloudStorageOutputWriter instance.
Args:
streaming_buffer: an instance of writable buffer from cloudstorage_api.
writer_spec: the specification for the writer.
"""
super(_GoogleCloudStorageRecordOutputWriter,
self).__init__(streaming_buffer, writer_spec)
self._record_writer = records.RecordsWriter(
super(_GoogleCloudStorageRecordOutputWriter, self))
def __init__(self, writer):
self._writer = writer
self._record_writer = records.RecordsWriter(writer)