def test_expect_bytes_written_bytes_IO(self): byte_len = 64 Bytes_io = six.BytesIO() w = RecordWriter(Bytes_io) bytes_to_write = b"x" * byte_len w.write(bytes_to_write) self.assertEqual(len(Bytes_io.getvalue()), (8 + 4 + byte_len + 4)) # uint64+uint32+data+uint32
def test_record_immediate_read(self):
filename = os.path.join(self.get_temp_dir(), "record_immediate_read")
chunks_to_write = ["hello world{}".format(i).encode() for i in range(10)]
w = RecordWriter(open(filename, 'wb'))
r = PyRecordReader_New(filename)
with self.assertRaises(errors.OutOfRangeError):
r.GetNext()
for bytes in chunks_to_write:
w.write(bytes)
w.flush()
r.GetNext()
self.assertEqual(r.record(), bytes)
w.close()
def test_empty_record(self):
filename = os.path.join(self.get_temp_dir(), "empty_record")
w = RecordWriter(open(filename, "wb"))
bytes_to_write = b""
w.write(bytes_to_write)
w.close()
r = PyRecordReader_New(filename)
r.GetNext()
self.assertEqual(r.record(), bytes_to_write)
def test_expect_bytes_written(self):
filename = os.path.join(self.get_temp_dir(), "expect_bytes_written")
byte_len = 64
w = RecordWriter(open(filename, 'wb'))
bytes_to_write = b"x" * byte_len
w.write(bytes_to_write)
w.close()
with open(filename, 'rb') as f:
self.assertEqual(len(f.read()), (8 + 4 + byte_len + 4)) # uint64+uint32+data+uint32
def test_record_writer_roundtrip(self):
filename = os.path.join(self.get_temp_dir(), "record_writer_roundtrip")
w = RecordWriter(open(filename, 'wb'))
chunks_to_write = ["hello world{}".format(i).encode() for i in range(10)]
for bytes in chunks_to_write:
w.write(bytes)
w.close()
r = PyRecordReader_New(filename)
for bytes in chunks_to_write:
r.GetNext()
self.assertEqual(r.record(), bytes)
def test_record_writer_roundtrip(self):
filename = os.path.join(self.get_temp_dir(), "record_writer_roundtrip")
w = RecordWriter(open(filename, 'wb'))
bytes_to_write = b"hello world"
times_to_test = 50
for _ in range(times_to_test):
w.write(bytes_to_write)
w.close()
r = PyRecordReader_New(filename)
for i in range(times_to_test):
r.GetNext()
self.assertEqual(r.record(), bytes_to_write)
def __init__(self,
logdir,
max_queue_size=10,
flush_secs=120,
filename_suffix=""):
"""Creates a `EventFileWriter` and an event file to write to.
On construction the summary writer creates a new event file in `logdir`.
This event file will contain `Event` protocol buffers, which are written to
disk via the add_event method.
The other arguments to the constructor control the asynchronous writes to
the event file:
Args:
logdir: A string. Directory where event file will be written.
max_queue_size: Integer. Size of the queue for pending events and summaries.
flush_secs: Number. How often, in seconds, to flush the
pending events and summaries to disk.
"""
self._logdir = logdir
if not tf.io.gfile.exists(logdir):
tf.io.gfile.makedirs(logdir)
self._file_name = (os.path.join(
logdir,
"events.out.tfevents.%010d.%s.%s.%s" % (
time.time(),
socket.gethostname(),
os.getpid(),
_global_uid.get(),
),
) + filename_suffix) # noqa E128
self._general_file_writer = tf.io.gfile.GFile(self._file_name, "wb")
self._async_writer = _AsyncWriter(
RecordWriter(self._general_file_writer), max_queue_size,
flush_secs)
# Initialize an event instance.
_event = event_pb2.Event(wall_time=time.time(),
file_version="brain.Event:2")
self.add_event(_event)
self.flush()