def test_thrift_garbage():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 2))
fp.write('ab')
fp.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def test_thrift_garbage():
with EphemeralFile('w') as fp:
fn = fp.name
fp.write(struct.pack('>L', 2))
fp.write('ab')
fp.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
with pytest.raises(RecordIO.PrematureEndOfStream):
rr.read()
def wait_for_rc(checkpoint, timeout=5.0):
start = time.time()
trr = ThriftRecordReader(open(checkpoint), RunnerCkpt)
while time.time() < start + timeout:
record = trr.read()
if record and record.process_status and record.process_status.return_code is not None:
return record.process_status.return_code
else:
time.sleep(0.1)
def wait_for_rc(checkpoint, timeout=5.0):
start = time.time()
trr = ThriftRecordReader(open(checkpoint), RunnerCkpt)
while time.time() < start + timeout:
record = trr.read()
if record and record.process_status and record.process_status.return_code is not None:
return record.process_status.return_code
else:
time.sleep(0.1)
def test_basic_thriftrecordwriter_write():
test_string = StringType("hello world")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string
def test_basic_thriftrecordwriter_write():
test_string = StringType("hello world")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string
def test_thrift_recordwriter_type_mismatch():
test_string = StringType("hello world")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, IntType)
# This is a peculiar behavior of Thrift in that it just returns
# ThriftType() with no serialization applied
assert rr.read() == IntType()
def test_paranoid_thrift_append_framing():
test_string_1 = StringType("hello world")
test_string_2 = StringType("ahoy ahoy, bonjour")
with EphemeralFile('w') as fp:
fn = fp.name
ThriftRecordWriter.append(fn, test_string_1)
ThriftRecordWriter.append(fn, test_string_2)
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_thrift_recordwriter_type_mismatch():
test_string = StringType("hello world")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string)
rw.close()
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, IntType)
# This is a peculiar behavior of Thrift in that it just returns
# ThriftType() with no serialization applied
assert rr.read() == IntType()
def test_paranoid_thrift_append_framing():
test_string_1 = StringType("hello world")
test_string_2 = StringType("ahoy ahoy, bonjour")
with EphemeralFile('w') as fp:
fn = fp.name
ThriftRecordWriter.append(fn, test_string_1)
ThriftRecordWriter.append(fn, test_string_2)
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_thriftrecordwriter_framing():
test_string_1 = StringType("hello world")
test_string_2 = StringType("ahoy ahoy, bonjour")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = ThriftRecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
def test_thriftrecordwriter_framing():
test_string_1 = StringType("hello world")
test_string_2 = StringType("ahoy ahoy, bonjour")
with EphemeralFile('w') as fp:
fn = fp.name
rw = ThriftRecordWriter(fp)
rw.write(test_string_1)
rw.close()
with open(fn, 'a') as fpa:
rw = ThriftRecordWriter(fpa)
rw.write(test_string_2)
with open(fn) as fpr:
rr = ThriftRecordReader(fpr, StringType)
assert rr.read() == test_string_1
assert rr.read() == test_string_2
