def test_basic_recordwriter_write_synced(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
RecordWriter.do_write(fp, test_string, StringCodec(), sync=True)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
RecordWriter.do_write(fp, test_string, StringCodec(), sync=True)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_recordreader_fails_with_writeonly():
with EphemeralFile('a') as fp:
with pytest.raises(RecordIO.InvalidFileHandle):
RecordReader(fp)
with EphemeralFile('w') as fp:
with pytest.raises(RecordIO.InvalidFileHandle):
RecordReader(fp)
def test_basic_recordwriter_write(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
rw = RecordWriter(fp)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
rw = RecordWriter(fp)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
RecordWriter.do_write(fp, test_string, RecordIO.StringCodec(), sync=True)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_premature_end_of_stream(self):
with self.EphemeralFile('r+') as fp:
fpr = FileLike.get(fp)
fpr = fp
fpr.write(struct.pack('>L', 1))
fpr.seek(0)
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_premature_end_of_stream(self):
with self.EphemeralFile('r+') as fp:
fpr = FileLike.get(fp)
fpr = fp
fpr.write(struct.pack('>L', 1))
fpr.seek(0)
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_basic_recordwriter_write():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordwriter_write():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_recordreader_works_with_plus():
with EphemeralFile('a+') as fp:
try:
RecordReader(fp)
except:
assert False, 'Failed to initialize RecordWriter in r+ mode'
with EphemeralFile('w+') as fp:
try:
RecordReader(fp)
except:
assert False, 'Failed to initialize RecordWriter in r+ mode'
def test_record_too_large(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE + 1))
fpw.write('a')
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_record_too_large(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE+1))
fpw.write('a')
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_premature_end_of_stream():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 1))
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_premature_end_of_stream():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 1))
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_sanity_check_bytes():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', RecordIO.SANITY_CHECK_BYTES+1))
fp.write('a')
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_sanity_check_bytes():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', RecordIO.SANITY_CHECK_BYTES + 1))
fp.write('a')
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_basic_recordwriter_write_synced_raises(self):
test_string = "hello world"
self.mox.StubOutWithMock(os, 'fsync')
with RecordioTestBase.EphemeralFile('r+') as fp:
os.fsync(fp.fileno()).AndRaise(OSError)
self.mox.ReplayAll()
rw = RecordWriter(FileLike(fp))
rw.set_sync(True)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced_raises(self):
test_string = "hello world"
self.mox.StubOutWithMock(os, 'fsync')
with RecordioTestBase.EphemeralFile('r+') as fp:
os.fsync(fp.fileno()).AndRaise(OSError)
self.mox.ReplayAll()
rw = RecordWriter(FileLike(fp))
rw.set_sync(True)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_bad_header_size(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE))
fpw._fp.truncate(RecordIO.RECORD_HEADER_SIZE - 1)
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
assert fpw.tell() != 0
fpw.seek(0)
assert rr.try_read() is None
assert fpw.tell() == 0
def test_bad_header_size(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE))
fpw._fp.truncate(RecordIO.RECORD_HEADER_SIZE - 1)
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
assert fpw.tell() != 0
fpw.seek(0)
assert rr.try_read() is None
assert fpw.tell() == 0
def test_paranoid_append_framing(self):
with self.DurableFile('w') as fp:
fn = fp.name
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
RecordWriter.append(fn, test_string_1)
RecordWriter.append(fn, test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
os.remove(fn)
def test_paranoid_append_framing(self):
with self.DurableFile('w') as fp:
fn = fp.name
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
RecordWriter.append(fn, test_string_1)
RecordWriter.append(fn, test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
os.remove(fn)
def test_basic_recordreader_iterator(self):
test_strings = ["hello", "world", "etc"]
with self.EphemeralFile('r+') as fp:
for string in test_strings:
RecordWriter.do_write(fp, string, StringCodec(), sync=True)
fp.seek(0)
rr = RecordReader(fp)
assert list(rr) == test_strings
def test_recordreader_works_with_plus(self):
for mode in ('a+', 'w+'):
with self.EphemeralFile(mode) as fp:
try:
RecordReader(fp)
except Exception as e:
assert False, (
"Failed to initialize RecordReader in '%s' mode (exception: %s)" % (mode, e))
def test_recordwriter_framing(self):
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
with self.EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = RecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_recordwriter_framing(self):
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
with self.EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = RecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_basic_recordreader_dup_failure(self):
fp = self.mox.CreateMock(FileLike)
fp.mode = 'r+'
fp.Error = FileLike.Error
fp.dup().AndRaise(FileLike.Error)
self.mox.ReplayAll()
rr = RecordReader(fp)
assert list(rr) == []
self.mox.VerifyAll()
def test_basic_recordreader_iter_failure(self):
self.mox.StubOutWithMock(RecordIO.Reader, 'do_read')
fp = self.mox.CreateMock(FileLike)
fp.mode = 'r+'
fp.dup().AndReturn(fp)
RecordIO.Reader.do_read(fp, mox.IsA(StringCodec)).AndRaise(RecordIO.Error)
fp.close()
self.mox.ReplayAll()
rr = RecordReader(fp)
assert list(rr) == []
def test_basic_recordreader_read(self):
test_string = "hello world"
with self.EphemeralFile('r') as fp:
fn = fp.name
rr = RecordReader(fp)
assert rr.read() is None
rr.close()
with open(fn, 'w') as fpw:
rw = RecordWriter(fpw)
rw.write(test_string)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordreader_read(self):
test_string = "hello world"
with self.EphemeralFile('r') as fp:
fn = fp.name
rr = RecordReader(fp)
assert rr.read() is None
rr.close()
with open(fn, 'w') as fpw:
rw = RecordWriter(fpw)
rw.write(test_string)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_raises_if_initialized_with_nil_filehandle(self):
with pytest.raises(RecordIO.InvalidFileHandle):
RecordWriter(None)
with pytest.raises(RecordIO.InvalidFileHandle):
RecordReader(None)
def test_raises_on_nonfile():
with pytest.raises(RecordIO.InvalidFileHandle):
RecordWriter('/tmp/poop')
with pytest.raises(RecordIO.InvalidFileHandle):
RecordReader('/tmp/poop')
def test_recordreader_fails_with_writeonly(self):
for mode in ('a', 'w'):
with self.EphemeralFile(mode) as fp:
with pytest.raises(RecordIO.InvalidFileHandle):
RecordReader(fp)
