Exemple #1
0
 def test_degradation_no_io(self):
     '''
     No disk IO when input fits in buffer 
     '''
     buffer_size = 10
     counters.reset_counter('load')
     self.validate_group_by(xrange(buffer_size), buffer_size=buffer_size)
     self.assertEqual(counters.get_counter('load'), 0, 'No disk IO')
 def testBasic(self):
     iterable = [{1: 2}, 'two', (3, 3, 3)]
     fileIterable = FileBackedIterable(iterable)
     fileIterator = iter(fileIterable)
     counters.reset_counter('load')
     for i in iterable:
         assert fileIterator.hasMore()
         iFile = next(fileIterator)
         assert i == iFile
     assert not fileIterator.hasMore()
     self.assertEqual(counters.get_counter('load'), 3)
     fileIterable.removeFile()
Exemple #3
0
    def test_degradation_linear(self):
        '''
        As the input size increases, the number of calls
        to "load" (disk IO) increases linearly to the input size 
        '''
        buffer_size = 10
        input_size1 = 2*buffer_size
        input_size2 = 10*buffer_size
        counters.set_counter('item_buffer', settings.MAX_CHUNKS)

        counters.reset_counter('load')
        self.validate_group_by(xrange(input_size1), buffer_size=buffer_size)
        load_to_input_size1 = 1.0*counters.get_counter('load') / input_size1

        counters.reset_counter('load')
        self.validate_group_by(xrange(input_size2), buffer_size=buffer_size)
        load_to_input_size2 = 1.0*counters.get_counter('load') / input_size2
        self.assertLess(input_size1, input_size2, 'test different input size')
        self.assertLessEqual(load_to_input_size1, load_to_input_size2,
                       '#load is linear to input size %s %s' % (load_to_input_size1, load_to_input_size2))