def test_source_iterator(self): class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: self.first = False else: for k in range(999): yield k for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1} ) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False for x, y in zip(xrange(1002), (a for a in fts_app.source_iterator())): if x == 0: self.assertTrue(y is None) elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True self.assertEqual(x, y, 'iterator fails on iteration %d' % x) else: print x, y if y is not None and not error_detected: error_detected = True self.assertTrue(x is None, 'iterator fails on iteration %d' % x)
def test_source_iterator(self): class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: self.first = False else: for k in range(999): yield k for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1} ) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False for x, y in zip(xrange(1002), (a for a in fts_app.source_iterator())): if x == 0: self.assertTrue(y is None) elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True self.assertEqual(x, y, 'iterator fails on iteration %d' % x) else: if y is not None and not error_detected: error_detected = True self.assertTrue(x is None, 'iterator fails on iteration %d' % x)
def test_source_iterator(self): faked_finished_func = Mock() class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: # make the iterator act as if exhausted on the very # first try self.first = False else: for k in range(999): # ensure that both forms (a single value or the # (args, kwargs) form are accepted.) if k % 4: yield k else: yield ( (k, ), {"finished_func": faked_finished_func} ) for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = get_logger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False no_finished_function_counter = 0 for x, y in zip(range(1002), (a for a in fts_app.source_iterator())): if x == 0: # the iterator is exhausted on the 1st try and should have # yielded a None before starting over assert y is None elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True assert x == y, 'iterator fails on iteration %d: %s' % (x, y) # invoke that finished func to ensure that we've got the # right object try: y[1]['finished_func']() except KeyError: no_finished_function_counter += 1 else: if y is not None and not error_detected: error_detected = True assert x is None, 'iterator fails on iteration %d: %s' % (x, y) assert faked_finished_func.call_count == (999 - no_finished_function_counter)
def test_source_iterator(self): faked_finished_func = Mock() class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: # make the iterator act as if exhausted on the very # first try self.first = False else: for k in range(999): # ensure that both forms (a single value or the # (args, kwargs) form are accepted.) if k % 4: yield k else: yield ((k, ), { "finished_func": faked_finished_func }) for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False no_finished_function_counter = 0 for x, y in zip(range(1002), (a for a in fts_app.source_iterator())): if x == 0: # the iterator is exhausted on the 1st try and should have # yielded a None before starting over assert y is None elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True assert x == y, 'iterator fails on iteration %d: %s' % (x, y) # invoke that finished func to ensure that we've got the # right object try: y[1]['finished_func']() except KeyError: no_finished_function_counter += 1 else: if y is not None and not error_detected: error_detected = True assert x is None, 'iterator fails on iteration %d: %s' % ( x, y) assert faked_finished_func.call_count == (999 - no_finished_function_counter)
def test_queue_iterator(self): faked_finished_func = Mock() class FakeQueue(object): def __init__(self): self.first = True def new_crashes(self): if self.first: # make the iterator act as if exhausted on the very # first try self.first = False else: for k in range(999): # ensure that both forms (a single value or the # (args, kwargs) form are accepted.) if k % 4: yield k else: yield ((k, ), { "finished_func": faked_finished_func }) for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump config = DotDict({ "logger": MagicMock(), "number_of_threads": 2, "maximum_queue_size": 2, "queue": DotDict({"crashqueue_class": FakeQueue}), "source": DotDict({"crashstorage_class": None}), "destination": DotDict({"crashstorage_class": FakeStorageDestination}), "producer_consumer": DotDict({ "producer_consumer_class": ThreadedTaskManager, "logger": MagicMock(), "number_of_threads": 1, "maximum_queue_size": 1, }), }) fts_app = FetchTransformSaveApp(config) fts_app.queue = FakeQueue() fts_app.source = None fts_app.destination = FakeStorageDestination error_detected = False no_finished_function_counter = 0 for x, y in zip(range(1002), (a for a in fts_app.source_iterator())): if x == 0: # the iterator is exhausted on the 1st try and should have # yielded a None before starting over assert y is None elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True assert x == y, "iterator fails on iteration %d: %s" % (x, y) # invoke that finished func to ensure that we've got the # right object try: y[1]["finished_func"]() except KeyError: no_finished_function_counter += 1 else: if y is not None and not error_detected: error_detected = True assert x is None, "iterator fails on iteration %d: %s" % ( x, y) assert faked_finished_func.call_count == (999 - no_finished_function_counter)