def test_update_str(self): opcounts = OperationCounters(CounterFactory(), 'some-name', coders.PickleCoder(), 0) self.verify_counters(opcounts, 0) opcounts.update_from(GlobalWindows.WindowedValue('abcde')) opcounts.update_collect() self.verify_counters(opcounts, 1)
def test_should_sample(self): # Order of magnitude more buckets than highest constant in code under test. buckets = [0] * 300 # The seed is arbitrary and exists just to ensure this test is robust. # If you don't like this seed, try your own; the test should still pass. random.seed(1717) # Do enough runs that the expected hits even in the last buckets # is big enough to expect some statistical smoothing. total_runs = 10 * len(buckets) # Fill the buckets. for _ in xrange(total_runs): opcounts = OperationCounters(CounterFactory(), 'some-name', coders.PickleCoder(), 0) for i in xrange(len(buckets)): if opcounts.should_sample(): buckets[i] += 1 # Look at the buckets to see if they are likely. for i in xrange(10): self.assertEqual(total_runs, buckets[i]) for i in xrange(10, len(buckets)): self.assertTrue( buckets[i] > 7 * total_runs / i, 'i=%d, buckets[i]=%d, expected=%d, ratio=%f' % (i, buckets[i], 10 * total_runs / i, buckets[i] / (10.0 * total_runs / i))) self.assertTrue( buckets[i] < 14 * total_runs / i, 'i=%d, buckets[i]=%d, expected=%d, ratio=%f' % (i, buckets[i], 10 * total_runs / i, buckets[i] / (10.0 * total_runs / i)))
def test_update_old_object(self): opcounts = OperationCounters(CounterFactory(), 'some-name', coders.PickleCoder(), 0) self.verify_counters(opcounts, 0) obj = OldClassThatDoesNotImplementLen() opcounts.update_from(GlobalWindows.WindowedValue(obj)) opcounts.update_collect() self.verify_counters(opcounts, 1)
def test_update_multiple(self): opcounts = OperationCounters(CounterFactory(), 'some-name', coders.PickleCoder(), 0) self.verify_counters(opcounts, 0) opcounts.update_from(GlobalWindows.windowed_value('abcde')) opcounts.update_from(GlobalWindows.windowed_value('defghij')) opcounts.update_collect() self.verify_counters(opcounts, 2) opcounts.update_from(GlobalWindows.windowed_value('klmnop')) opcounts.update_collect() self.verify_counters(opcounts, 3)
class MapTask(object): """A map task decoded into operations and ready to be executed. Attributes: operations: A list of Worker* object created by parsing the instructions within the map task. stage_name: The name of this map task execution stage. step_names: The names of the step corresponding to each map task operation. """ def __init__(self, operations, stage_name, step_names): self.operations = operations self.stage_name = stage_name self.step_names = step_names self.counter_factory = CounterFactory() def itercounters(self): return self.counter_factory.get_counters() def __str__(self): return '<%s %s steps=%s>' % (self.__class__.__name__, self.stage_name, '+'.join(self.step_names))
class MapTask(object): """A map task decoded into operations and ready to be executed. Attributes: operations: A list of Worker* object created by parsing the instructions within the map task. stage_name: The name of this map task execution stage. step_names: The names of the step corresponding to each map task operation. """ def __init__(self, operations, stage_name, step_names): self.operations = operations self.stage_name = stage_name self.step_names = step_names self.counter_factory = CounterFactory() def itercounters(self): return self.counter_factory.get_counters() def __str__(self): return '<%s %s steps=%s>' % (self.__class__.__name__, self.stage_name, '+'.join(self.step_names))
def __init__(self, operations, stage_name, step_names): self.operations = operations self.stage_name = stage_name self.step_names = step_names self.counter_factory = CounterFactory()