def test_pseudo_record(self):
sio = StringIO()
pre_data = [1,1,2,3,5]
post_data = [8,13,21]
for i in pre_data:
with RecordWriter(sio) as r:
r.write(str(i))
sio.write(SENTINEL + cPickle.dumps(20)) # fake record
sio.write(random_string(20))
sio.write(cPickle.dumps(hash(''))) # wrong hash
for i in post_data:
with RecordWriter(sio) as r:
r.write(str(i))
sio.seek(0)
read_data = [int(s) for s in RecordReader(sio, tolerate_subsequent_error=True)]
self.assertEqual(read_data, [1,1,2,3,5,8,13,21])
def test_recordreader(self):
stream = StringIO()
write_data = [cPickle.dumps(random.random()) for _ in xrange(2)]
write_data.append('abc12#jeoht38#SoSooihetS#') # contains sentinel
write_data.extend(random_string(8) for _ in xrange(2))
for i in write_data:
with RecordWriter(stream) as r:
r.write(i) # write each obj as its own record
size = stream.tell()
# reading from the beginning gets all values
stream.seek(0)
read_data = list(RecordReader(stream))
self.assertEqual(len(write_data), len(read_data))
self.assertEqual(write_data, read_data)
# past the beginning gets fewer values
last_count = len(read_data)
for offset in xrange(1, size):
stream.seek(offset, os.SEEK_SET)
values = list(RecordReader(stream))
self.assertTrue(len(values) == last_count or len(values) == last_count-1)
last_count = min(last_count, len(values))
# truncated at both ends, shrinking window
stream.seek(0)
last_count = len(read_data)
value_counts = set()
for offset in xrange(1, size/2):
ranger = RangeReader(stream, start=offset, end=size-offset, rebase=True, hard_end=True)
values = list(RecordReader(ranger, tolerate_subsequent_error=True))
self.assertTrue(len(values) <= last_count)
value_counts.add(len(values))
last_count = min(last_count, len(values))
self.assertEqual(len(value_counts), 4)
def def_rfunc(): return randomtools.random_string(8) rfunc = rfunc or def_rfunc