def test_update_str(self):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
self.verify_counters(opcounts, 0)
opcounts.update_from(GlobalWindows.WindowedValue('abcde'))
opcounts.update_collect()
self.verify_counters(opcounts, 1)
def test_should_sample(self):
# Order of magnitude more buckets than highest constant in code under test.
buckets = [0] * 300
# The seed is arbitrary and exists just to ensure this test is robust.
# If you don't like this seed, try your own; the test should still pass.
random.seed(1717)
# Do enough runs that the expected hits even in the last buckets
# is big enough to expect some statistical smoothing.
total_runs = 10 * len(buckets)
# Fill the buckets.
for _ in xrange(total_runs):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
for i in xrange(len(buckets)):
if opcounts.should_sample():
buckets[i] += 1
# Look at the buckets to see if they are likely.
for i in xrange(10):
self.assertEqual(total_runs, buckets[i])
for i in xrange(10, len(buckets)):
self.assertTrue(
buckets[i] > 7 * total_runs / i,
'i=%d, buckets[i]=%d, expected=%d, ratio=%f' %
(i, buckets[i], 10 * total_runs / i, buckets[i] /
(10.0 * total_runs / i)))
self.assertTrue(
buckets[i] < 14 * total_runs / i,
'i=%d, buckets[i]=%d, expected=%d, ratio=%f' %
(i, buckets[i], 10 * total_runs / i, buckets[i] /
(10.0 * total_runs / i)))
def test_update_old_object(self):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
self.verify_counters(opcounts, 0)
obj = OldClassThatDoesNotImplementLen()
opcounts.update_from(GlobalWindows.WindowedValue(obj))
opcounts.update_collect()
self.verify_counters(opcounts, 1)
def test_update_multiple(self):
opcounts = OperationCounters(CounterFactory(), 'some-name',
coders.PickleCoder(), 0)
self.verify_counters(opcounts, 0)
opcounts.update_from(GlobalWindows.windowed_value('abcde'))
opcounts.update_from(GlobalWindows.windowed_value('defghij'))
opcounts.update_collect()
self.verify_counters(opcounts, 2)
opcounts.update_from(GlobalWindows.windowed_value('klmnop'))
opcounts.update_collect()
self.verify_counters(opcounts, 3)
class MapTask(object):
"""A map task decoded into operations and ready to be executed.
Attributes:
operations: A list of Worker* object created by parsing the instructions
within the map task.
stage_name: The name of this map task execution stage.
step_names: The names of the step corresponding to each map task operation.
"""
def __init__(self, operations, stage_name, step_names):
self.operations = operations
self.stage_name = stage_name
self.step_names = step_names
self.counter_factory = CounterFactory()
def itercounters(self):
return self.counter_factory.get_counters()
def __str__(self):
return '<%s %s steps=%s>' % (self.__class__.__name__, self.stage_name,
'+'.join(self.step_names))
class MapTask(object):
"""A map task decoded into operations and ready to be executed.
Attributes:
operations: A list of Worker* object created by parsing the instructions
within the map task.
stage_name: The name of this map task execution stage.
step_names: The names of the step corresponding to each map task operation.
"""
def __init__(self, operations, stage_name, step_names):
self.operations = operations
self.stage_name = stage_name
self.step_names = step_names
self.counter_factory = CounterFactory()
def itercounters(self):
return self.counter_factory.get_counters()
def __str__(self):
return '<%s %s steps=%s>' % (self.__class__.__name__, self.stage_name,
'+'.join(self.step_names))
def __init__(self, operations, stage_name, step_names): self.operations = operations self.stage_name = stage_name self.step_names = step_names self.counter_factory = CounterFactory()
