def test_recordreader_fails_with_writeonly():
with EphemeralFile('a') as fp:
with pytest.raises(RecordIO.InvalidFileHandle):
RecordReader(fp)
with EphemeralFile('w') as fp:
with pytest.raises(RecordIO.InvalidFileHandle):
RecordReader(fp)
def test_recordreader_works_with_plus():
with EphemeralFile('a+') as fp:
try:
RecordReader(fp)
except:
assert False, 'Failed to initialize RecordWriter in r+ mode'
with EphemeralFile('w+') as fp:
try:
RecordReader(fp)
except:
assert False, 'Failed to initialize RecordWriter in r+ mode'
def test_basic_recordwriter_write_synced():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
RecordWriter.do_write(fp, test_string, RecordIO.StringCodec(), sync=True)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_basic_recordwriter_write():
test_string = "hello world"
with EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string
def test_premature_end_of_stream():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 1))
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_thrift_garbage():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 2))
fp.write('ab')
fp.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_sanity_check_bytes():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', RecordIO.SANITY_CHECK_BYTES + 1))
fp.write('a')
fp.close()
with open(fn) as fpr:
rr = RecordReader(fpr)
with pytest.raises(RecordIO.RecordSizeExceeded):
rr.read()
def test_basic_thriftrecordwriter_write():
test_string = StringType("hello world")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string
def test_thrift_recordwriter_type_mismatch():
test_string = StringType("hello world")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, IntType)
# This is a peculiar behavior of Thrift in that it just returns
# ThriftType() with no serialization applied
assert rr.read() == IntType()
def test_paranoid_thrift_append_framing():
test_string_1 = StringType("hello world")
test_string_2 = StringType("ahoy ahoy, bonjour")
with EphemeralFile('w') as fp:
fn = fp.name
ThriftRecordWriter.append(fn, test_string_1)
ThriftRecordWriter.append(fn, test_string_2)
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_basic_recordreader_try_read():
test_string = "hello world"
with EphemeralFile('r') as fp:
fn = fp.name
rr = RecordReader(fp)
assert rr.try_read() is None
rr.close()
with open(fn, 'w') as fpw:
rw = RecordWriter(fpw)
rw.write(test_string)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.try_read() == test_string
def test_recordwriter_framing():
test_string_1 = "hello world"
test_string_2 = "ahoy ahoy, bonjour"
with EphemeralFile('w') as fp:
fn = fp.name
rw = RecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = RecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = RecordReader(fpr)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_thriftrecordreader_iteration():
test_string_1 = StringType("hello world")
test_string_2 = StringType("ahoy ahoy, bonjour")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string_1)
rw.write(test_string_2)
rw.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
records = []
for record in rr:
records.append(record)
assert records == [test_string_1, test_string_2]
def test_thriftrecordwriter_framing():
test_string_1 = StringType("hello world")
test_string_2 = StringType("ahoy ahoy, bonjour")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = ThriftRecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_recordwriter_works_with_write():
with EphemeralFile('w') as fp:
try:
RecordWriter(fp)
except:
assert False, 'Failed to initialize RecordWriter in r+ mode'
def test_recordwriter_raises_on_readonly_file():
with EphemeralFile('r') as fp:
with pytest.raises(RecordIO.InvalidFileHandle):
RecordWriter(fp)
def test_thrift_invalid_codec_with_object_instead_of_class():
with EphemeralFile('w') as fp:
with pytest.raises(ThriftRecordIO.InvalidThriftException):
ThriftRecordReader(fp, StringType())
def test_thrift_invalid_codec_with_nonclass():
with EphemeralFile('w') as fp:
with pytest.raises(ThriftRecordIO.InvalidThriftException):
ThriftRecordReader(fp, 5)
