def test_premature_end_of_stream(self):
with self.EphemeralFile('r+') as fp:
fpr = FileLike.get(fp)
fpr = fp
fpr.write(struct.pack('>L', 1))
fpr.seek(0)
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_premature_end_of_stream(self):
with self.EphemeralFile('r+') as fp:
fpr = FileLike.get(fp)
fpr = fp
fpr.write(struct.pack('>L', 1))
fpr.seek(0)
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_premature_end_of_stream():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 1))
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_record_too_large(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE + 1))
fpw.write('a')
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_record_too_large(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE+1))
fpw.write('a')
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_premature_end_of_stream():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 1))
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_sanity_check_bytes():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', RecordIO.SANITY_CHECK_BYTES + 1))
fp.write('a')
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_sanity_check_bytes():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', RecordIO.SANITY_CHECK_BYTES+1))
fp.write('a')
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_bad_header_size(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE))
fpw._fp.truncate(RecordIO.RECORD_HEADER_SIZE - 1)
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
assert fpw.tell() != 0
fpw.seek(0)
assert rr.try_read() is None
assert fpw.tell() == 0
def test_bad_header_size(self):
with self.EphemeralFile('r+') as fp:
fpw = FileLike.get(fp)
fpw.write(struct.pack('>L', RecordIO.MAXIMUM_RECORD_SIZE))
fpw._fp.truncate(RecordIO.RECORD_HEADER_SIZE - 1)
fpw.flush()
fpw.seek(0)
rr = RecordReader(fp)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
assert fpw.tell() != 0
fpw.seek(0)
assert rr.try_read() is None
assert fpw.tell() == 0
def test_basic_recordreader_read(self):
test_string = "hello world"
with self.EphemeralFile('r') as fp:
fn = fp.name
rr = RecordReader(fp)
assert rr.read() is None
rr.close()
with open(fn, 'w') as fpw:
rw = RecordWriter(fpw)
rw.write(test_string)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
RecordWriter.do_write(fp, test_string, StringCodec(), sync=True)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
RecordWriter.do_write(fp, test_string, StringCodec(), sync=True)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordreader_read(self):
test_string = "hello world"
with self.EphemeralFile('r') as fp:
fn = fp.name
rr = RecordReader(fp)
assert rr.read() is None
rr.close()
with open(fn, 'w') as fpw:
rw = RecordWriter(fpw)
rw.write(test_string)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
RecordWriter.do_write(fp, test_string, RecordIO.StringCodec(), sync=True)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordwriter_write(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
rw = RecordWriter(fp)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write(self):
test_string = "hello world"
with self.EphemeralFile('r+') as fp:
rw = RecordWriter(fp)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordwriter_write():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced_raises(self):
test_string = "hello world"
self.mox.StubOutWithMock(os, 'fsync')
with RecordioTestBase.EphemeralFile('r+') as fp:
os.fsync(fp.fileno()).AndRaise(OSError)
self.mox.ReplayAll()
rw = RecordWriter(FileLike(fp))
rw.set_sync(True)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_basic_recordwriter_write_synced_raises(self):
test_string = "hello world"
self.mox.StubOutWithMock(os, 'fsync')
with RecordioTestBase.EphemeralFile('r+') as fp:
os.fsync(fp.fileno()).AndRaise(OSError)
self.mox.ReplayAll()
rw = RecordWriter(FileLike(fp))
rw.set_sync(True)
rw.write(test_string)
fp.seek(0)
rr = RecordReader(fp)
assert rr.read() == test_string
def test_paranoid_append_framing(self):
with self.DurableFile('w') as fp:
fn = fp.name
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
RecordWriter.append(fn, test_string_1)
RecordWriter.append(fn, test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
os.remove(fn)
def test_paranoid_append_framing(self):
with self.DurableFile('w') as fp:
fn = fp.name
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
RecordWriter.append(fn, test_string_1)
RecordWriter.append(fn, test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
os.remove(fn)
def test_recordwriter_framing(self):
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
with self.EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = RecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_recordwriter_framing(self):
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
with self.EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = RecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2