def test_async_writer_without_write(self): logdir = self.get_temp_dir() w = EventFileWriter(logdir) w.close() event_files = sorted(glob.glob(os.path.join(logdir, '*'))) r = PyRecordReader_New(event_files[0]) r.GetNext() s = event_pb2.Event.FromString(r.record()) self.assertEqual(s.file_version, "brain.Event:2")
def test_empty_record(self): filename = os.path.join(self.get_temp_dir(), "empty_record") w = RecordWriter(open(filename, "wb")) bytes_to_write = b"" w.write(bytes_to_write) w.close() r = PyRecordReader_New(filename) r.GetNext() self.assertEqual(r.record(), bytes_to_write)
def test_record_writer_roundtrip(self): filename = os.path.join(self.get_temp_dir(), "record_writer_roundtrip") w = RecordWriter(open(filename, 'wb')) chunks_to_write = ["hello world{}".format(i).encode() for i in range(10)] for bytes in chunks_to_write: w.write(bytes) w.close() r = PyRecordReader_New(filename) for bytes in chunks_to_write: r.GetNext() self.assertEqual(r.record(), bytes)
def test_record_writer_roundtrip(self): filename = os.path.join(self.get_temp_dir(), "record_writer_roundtrip") w = RecordWriter(filename) bytes_to_write = b"hello world" times_to_test = 50 for _ in range(times_to_test): w.write(bytes_to_write) w.close() r = PyRecordReader_New(filename) for i in range(times_to_test): r.GetNext() self.assertEqual(r.record(), bytes_to_write)
def test_record_immediate_read(self): filename = os.path.join(self.get_temp_dir(), "record_immediate_read") chunks_to_write = ["hello world{}".format(i).encode() for i in range(10)] w = RecordWriter(open(filename, 'wb')) r = PyRecordReader_New(filename) with self.assertRaises(errors.OutOfRangeError): r.GetNext() for bytes in chunks_to_write: w.write(bytes) w.flush() r.GetNext() self.assertEqual(r.record(), bytes) w.close()
def test_event_file_writer_roundtrip(self): _TAGNAME = 'dummy' _DUMMY_VALUE = 42 logdir = self.get_temp_dir() w = EventFileWriter(logdir) summary = Summary(value=[Summary.Value(tag=_TAGNAME, simple_value=_DUMMY_VALUE)]) fakeevent = event_pb2.Event(summary=summary) w.add_event(fakeevent) w.close() event_files = sorted(glob.glob(os.path.join(logdir, '*'))) self.assertEqual(len(event_files), 1) r = PyRecordReader_New(event_files[0]) r.GetNext() # meta data, so skip r.GetNext() self.assertEqual(fakeevent.SerializeToString(), r.record())
def test_writer(self): TEST_LEN = 100 N_PROC = 4 writer = SummaryWriter() event_filename = writer.file_writer.event_writer._ev_writer._file_name predifined_values = list(range(TEST_LEN)) def train3(): for i in range(TEST_LEN): writer.add_scalar('many_write_in_func', predifined_values[i]) time.sleep(0.01 * np.random.randint(0, 10)) processes = [] for i in range(N_PROC): p1 = mp.Process(target=train3) processes.append(p1) p1.start() for p in processes: p.join() writer.close() collected_values = [] r = PyRecordReader_New(event_filename) r.GetNext() # meta data, so skip for _ in range(TEST_LEN * N_PROC): # all of the data should be flushed r.GetNext() ev = event_pb2.Event() value = ev.FromString(r.record()).summary.value collected_values.append(value[0].simple_value) collected_values = sorted(collected_values) for i in range(TEST_LEN): for j in range(N_PROC): assert collected_values[i * N_PROC + j] == i