Example #1
0
 def test_pseudo_record(self):
     sio = StringIO()
     pre_data = [1,1,2,3,5]
     post_data = [8,13,21]
     for i in pre_data:
         with RecordWriter(sio) as r:
             r.write(str(i))
     sio.write(SENTINEL + cPickle.dumps(20)) # fake record
     sio.write(random_string(20))
     sio.write(cPickle.dumps(hash(''))) # wrong hash
     for i in post_data:
         with RecordWriter(sio) as r:
             r.write(str(i))
     sio.seek(0)
     read_data = [int(s) for s in RecordReader(sio, tolerate_subsequent_error=True)]
     self.assertEqual(read_data, [1,1,2,3,5,8,13,21])
Example #2
0
    def test_recordreader(self):
        stream = StringIO()
        write_data = [cPickle.dumps(random.random()) for _ in xrange(2)]
        write_data.append('abc12#jeoht38#SoSooihetS#') # contains sentinel
        write_data.extend(random_string(8) for _ in xrange(2))
        for i in write_data:
            with RecordWriter(stream) as r:
                r.write(i) # write each obj as its own record

        size = stream.tell()

        # reading from the beginning gets all values
        stream.seek(0)
        read_data = list(RecordReader(stream))
        self.assertEqual(len(write_data), len(read_data))
        self.assertEqual(write_data, read_data)

        # past the beginning gets fewer values
        last_count = len(read_data)
        for offset in xrange(1, size):
            stream.seek(offset, os.SEEK_SET)
            values = list(RecordReader(stream))
            self.assertTrue(len(values) == last_count or len(values) == last_count-1)
            last_count = min(last_count, len(values))

        # truncated at both ends, shrinking window
        stream.seek(0)
        last_count = len(read_data)
        value_counts = set()
        for offset in xrange(1, size/2):
            ranger = RangeReader(stream, start=offset, end=size-offset, rebase=True, hard_end=True)
            values = list(RecordReader(ranger, tolerate_subsequent_error=True))
            self.assertTrue(len(values) <= last_count)
            value_counts.add(len(values))
            last_count = min(last_count, len(values))
        self.assertEqual(len(value_counts), 4)
Example #3
0
 def def_rfunc(): return randomtools.random_string(8)
 rfunc = rfunc or def_rfunc