def test_update_int(self):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
self.verify_counters(opcounts, 0)
opcounts.update_from(GlobalWindows.windowed_value(1))
opcounts.update_collect()
self.verify_counters(opcounts, 1)
def test_should_sample(self):
# Order of magnitude more buckets than highest constant in code under test.
buckets = [0] * 300
# The seed is arbitrary and exists just to ensure this test is robust.
# If you don't like this seed, try your own; the test should still pass.
random.seed(1717)
# Do enough runs that the expected hits even in the last buckets
# is big enough to expect some statistical smoothing.
total_runs = 10 * len(buckets)
# Fill the buckets.
for _ in range(total_runs):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
for i in range(len(buckets)):
if opcounts.should_sample():
buckets[i] += 1
# Look at the buckets to see if they are likely.
for i in range(10):
self.assertEqual(total_runs, buckets[i])
for i in range(10, len(buckets)):
self.assertTrue(buckets[i] > 7 * total_runs / i,
'i=%d, buckets[i]=%d, expected=%d, ratio=%f' % (
i, buckets[i],
10 * total_runs / i,
buckets[i] / (10.0 * total_runs / i)))
self.assertTrue(buckets[i] < 14 * total_runs / i,
'i=%d, buckets[i]=%d, expected=%d, ratio=%f' % (
i, buckets[i],
10 * total_runs / i,
buckets[i] / (10.0 * total_runs / i)))
def test_should_sample(self):
# Order of magnitude more buckets than highest constant in code under test.
buckets = [0] * 300
# The seed is arbitrary and exists just to ensure this test is robust.
# If you don't like this seed, try your own; the test should still pass.
random.seed(1720)
# Do enough runs that the expected hits even in the last buckets
# is big enough to expect some statistical smoothing.
total_runs = 10 * len(buckets)
# Fill the buckets.
for _ in range(total_runs):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
for i in range(len(buckets)):
if opcounts.should_sample():
buckets[i] += 1
# Look at the buckets to see if they are likely.
for i in range(10):
self.assertEqual(total_runs, buckets[i])
for i in range(10, len(buckets)):
self.assertTrue(
buckets[i] > 7 * total_runs / i,
'i=%d, buckets[i]=%d, expected=%d, ratio=%f' %
(i, buckets[i], 10 * total_runs / i, buckets[i] /
(10.0 * total_runs / i)))
self.assertTrue(
buckets[i] < 14 * total_runs / i,
'i=%d, buckets[i]=%d, expected=%d, ratio=%f' %
(i, buckets[i], 10 * total_runs / i, buckets[i] /
(10.0 * total_runs / i)))
def test_update_str(self):
coder = coders.PickleCoder()
opcounts = OperationCounters(CounterFactory(), 'some-name', coder, 0)
self.verify_counters(opcounts, 0, float('nan'))
value = GlobalWindows.windowed_value('abcde')
opcounts.update_from(value)
estimated_size = coder.estimate_size(value)
self.verify_counters(opcounts, 1, estimated_size)
def test_update_str(self):
coder = coders.PickleCoder()
opcounts = OperationCounters(CounterFactory(), 'some-name',
coder, 0)
self.verify_counters(opcounts, 0, float('nan'))
value = GlobalWindows.windowed_value('abcde')
opcounts.update_from(value)
estimated_size = coder.estimate_size(value)
self.verify_counters(opcounts, 1, estimated_size)
def test_update_old_object(self):
coder = coders.PickleCoder()
opcounts = OperationCounters(CounterFactory(), 'some-name', coder, 0)
self.verify_counters(opcounts, 0, float('nan'))
obj = OldClassThatDoesNotImplementLen()
value = GlobalWindows.windowed_value(obj)
opcounts.update_from(value)
estimated_size = coder.estimate_size(value)
self.verify_counters(opcounts, 1, estimated_size)
def test_update_old_object(self):
coder = coders.PickleCoder()
opcounts = OperationCounters(CounterFactory(), 'some-name',
coder, 0)
self.verify_counters(opcounts, 0, float('nan'))
obj = OldClassThatDoesNotImplementLen()
value = GlobalWindows.windowed_value(obj)
opcounts.update_from(value)
estimated_size = coder.estimate_size(value)
self.verify_counters(opcounts, 1, estimated_size)
def test_update_int(self):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
self.verify_counters(opcounts, 0)
opcounts.update_from(GlobalWindows.windowed_value(1))
opcounts.update_collect()
self.verify_counters(opcounts, 1)
def test_update_multiple(self):
coder = coders.PickleCoder()
total_size = 0
opcounts = OperationCounters(CounterFactory(), 'some-name', coder, 0)
self.verify_counters(opcounts, 0, float('nan'))
value = GlobalWindows.windowed_value('abcde')
opcounts.update_from(value)
total_size += coder.estimate_size(value)
value = GlobalWindows.windowed_value('defghij')
opcounts.update_from(value)
total_size += coder.estimate_size(value)
self.verify_counters(opcounts, 2, float(total_size) / 2)
value = GlobalWindows.windowed_value('klmnop')
opcounts.update_from(value)
total_size += coder.estimate_size(value)
self.verify_counters(opcounts, 3, float(total_size) / 3)
def test_update_multiple(self):
coder = coders.PickleCoder()
total_size = 0
opcounts = OperationCounters(CounterFactory(), 'some-name',
coder, 0)
self.verify_counters(opcounts, 0, float('nan'))
value = GlobalWindows.windowed_value('abcde')
opcounts.update_from(value)
total_size += coder.estimate_size(value)
value = GlobalWindows.windowed_value('defghij')
opcounts.update_from(value)
total_size += coder.estimate_size(value)
self.verify_counters(opcounts, 2, (float(total_size) / 2))
value = GlobalWindows.windowed_value('klmnop')
opcounts.update_from(value)
total_size += coder.estimate_size(value)
self.verify_counters(opcounts, 3, (float(total_size) / 3))
def test_update_batch(self):
coder = coders.FastPrimitivesCoder()
opcounts = OperationCounters(
CounterFactory(),
'some-name',
coder,
0,
producer_batch_converter=typehints.batch.BatchConverter.
from_typehints(element_type=typehints.Any,
batch_type=typehints.List[typehints.Any]))
size_per_element = coder.estimate_size(50)
self.verify_counters(opcounts, 0, float('nan'))
opcounts.update_from_batch(
GlobalWindows.windowed_batch(list(range(100))))
self.verify_counters(opcounts, 100, size_per_element)
opcounts.update_from_batch(
GlobalWindows.windowed_batch(list(range(100, 200))))
self.verify_counters(opcounts, 200, size_per_element)
