def test_no_source(self): class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = get_logger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': None}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) with pytest.raises(TypeError): fts_app.main()
def test_source_iterator(self): class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: self.first = False else: for k in range(999): yield k for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1} ) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False for x, y in zip(xrange(1002), (a for a in fts_app.source_iterator())): if x == 0: self.assertTrue(y is None) elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True self.assertEqual(x, y, 'iterator fails on iteration %d' % x) else: print x, y if y is not None and not error_detected: error_detected = True self.assertTrue(x is None, 'iterator fails on iteration %d' % x)
def test_no_source(self): class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump config = DotDict({ 'logger': MagicMock(), 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': None}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': MagicMock(), 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) with pytest.raises(TypeError): fts_app.main()
def test_source_iterator(self): class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: self.first = False else: for k in range(999): yield k for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1} ) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False for x, y in zip(xrange(1002), (a for a in fts_app.source_iterator())): if x == 0: self.assertTrue(y is None) elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True self.assertEqual(x, y, 'iterator fails on iteration %d' % x) else: if y is not None and not error_detected: error_detected = True self.assertTrue(x is None, 'iterator fails on iteration %d' % x)
def test_no_destination(self): class FakeStorageSource(object): def __init__(self, config, quit_check_callback): self.store = DotDict({ '1234': DotDict({ 'ooid': '1234', 'Product': 'FireSquid', 'Version': '1.0' }), '1235': DotDict({ 'ooid': '1235', 'Product': 'ThunderRat', 'Version': '1.0' }), '1236': DotDict({ 'ooid': '1236', 'Product': 'Caminimal', 'Version': '1.0' }), '1237': DotDict({ 'ooid': '1237', 'Product': 'Fennicky', 'Version': '1.0' }), }) def get_raw_crash(self, ooid): return self.store[ooid] def get_raw_dump(self, ooid): return 'this is a fake dump' def new_ooids(self): for k in self.store.keys(): yield k logger = get_logger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': None}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) with pytest.raises(TypeError): fts_app.main()
def test_no_destination(self): class FakeStorageSource(object): def __init__(self, config, quit_check_callback): self.store = DotDict({ '1234': DotDict({ 'ooid': '1234', 'Product': 'FireSquid', 'Version': '1.0' }), '1235': DotDict({ 'ooid': '1235', 'Product': 'ThunderRat', 'Version': '1.0' }), '1236': DotDict({ 'ooid': '1236', 'Product': 'Caminimal', 'Version': '1.0' }), '1237': DotDict({ 'ooid': '1237', 'Product': 'Fennicky', 'Version': '1.0' }), }) def get_raw_crash(self, ooid): return self.store[ooid] def get_raw_dump(self, ooid): return 'this is a fake dump' def new_ooids(self): for k in self.store.keys(): yield k config = DotDict({ 'logger': MagicMock(), 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': None}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': MagicMock(), 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) with pytest.raises(TypeError): fts_app.main()
def test_no_source(self): class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump): self.store[raw_crash.ooid] = raw_crash self.dumps[raw_crash.ooid] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'source': DotDict({'crashstorage': None}), 'destination': DotDict({'crashstorage': FakeStorageDestination}) }) fts_app = FetchTransformSaveApp(config) self.assertRaises(TypeError, fts_app.main)
def test_no_destination(self): class FakeStorageSource(object): def __init__(self, config, quit_check_callback): self.store = DotDict({ '1234': DotDict({ 'ooid': '1234', 'Product': 'FireFloozy', 'Version': '1.0' }), '1235': DotDict({ 'ooid': '1235', 'Product': 'ThunderRat', 'Version': '1.0' }), '1236': DotDict({ 'ooid': '1236', 'Product': 'Caminimal', 'Version': '1.0' }), '1237': DotDict({ 'ooid': '1237', 'Product': 'Fennicky', 'Version': '1.0' }), }) def get_raw_crash(self, ooid): return self.store[ooid] def get_dump(self, ooid): return 'this is a fake dump' def new_ooids(self): for k in self.store.keys(): yield k logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'source': DotDict({'crashstorage': FakeStorageSource}), 'destination': DotDict({'crashstorage': None}) }) fts_app = FetchTransformSaveApp(config) self.assertRaises(TypeError, fts_app.main)
def test_no_source(self): class FakeStorageDestination(object): def __init__(self, config): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump config = DotDict({ "logger": MagicMock(), "number_of_threads": 2, "maximum_queue_size": 2, "queue": DotDict({"crashqueue_class": None}), "source": DotDict({"crashstorage_class": None}), "destination": DotDict({"crashstorage_class": FakeStorageDestination}), "producer_consumer": DotDict({ "producer_consumer_class": ThreadedTaskManager, "logger": MagicMock(), "number_of_threads": 1, "maximum_queue_size": 1, }), }) fts_app = FetchTransformSaveApp(config) with pytest.raises(TypeError): fts_app.main()
def test_source_iterator(self): faked_finished_func = Mock() class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: # make the iterator act as if exhausted on the very # first try self.first = False else: for k in range(999): # ensure that both forms (a single value or the # (args, kwargs) form are accepted.) if k % 4: yield k else: yield ( (k, ), {"finished_func": faked_finished_func} ) for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = get_logger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False no_finished_function_counter = 0 for x, y in zip(range(1002), (a for a in fts_app.source_iterator())): if x == 0: # the iterator is exhausted on the 1st try and should have # yielded a None before starting over assert y is None elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True assert x == y, 'iterator fails on iteration %d: %s' % (x, y) # invoke that finished func to ensure that we've got the # right object try: y[1]['finished_func']() except KeyError: no_finished_function_counter += 1 else: if y is not None and not error_detected: error_detected = True assert x is None, 'iterator fails on iteration %d: %s' % (x, y) assert faked_finished_func.call_count == (999 - no_finished_function_counter)
def test_source_iterator(self): faked_finished_func = Mock() class FakeStorageSource(object): def __init__(self): self.first = True def new_crashes(self): if self.first: # make the iterator act as if exhausted on the very # first try self.first = False else: for k in range(999): # ensure that both forms (a single value or the # (args, kwargs) form are accepted.) if k % 4: yield k else: yield ((k, ), { "finished_func": faked_finished_func }) for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config, quit_check_callback): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump logger = SilentFakeLogger() config = DotDict({ 'logger': logger, 'number_of_threads': 2, 'maximum_queue_size': 2, 'number_of_submissions': 'forever', 'source': DotDict({'crashstorage_class': FakeStorageSource}), 'destination': DotDict({'crashstorage_class': FakeStorageDestination}), 'producer_consumer': DotDict({ 'producer_consumer_class': ThreadedTaskManager, 'logger': logger, 'number_of_threads': 1, 'maximum_queue_size': 1 }) }) fts_app = FetchTransformSaveApp(config) fts_app.source = FakeStorageSource() fts_app.destination = FakeStorageDestination error_detected = False no_finished_function_counter = 0 for x, y in zip(range(1002), (a for a in fts_app.source_iterator())): if x == 0: # the iterator is exhausted on the 1st try and should have # yielded a None before starting over assert y is None elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True assert x == y, 'iterator fails on iteration %d: %s' % (x, y) # invoke that finished func to ensure that we've got the # right object try: y[1]['finished_func']() except KeyError: no_finished_function_counter += 1 else: if y is not None and not error_detected: error_detected = True assert x is None, 'iterator fails on iteration %d: %s' % ( x, y) assert faked_finished_func.call_count == (999 - no_finished_function_counter)
def test_no_destination(self): class FakeStorageSource(object): def __init__(self, config): self.store = DotDict({ "1234": DotDict({ "ooid": "1234", "Product": "FireSquid", "Version": "1.0" }), "1235": DotDict({ "ooid": "1235", "Product": "ThunderRat", "Version": "1.0" }), "1236": DotDict({ "ooid": "1236", "Product": "Caminimal", "Version": "1.0" }), "1237": DotDict({ "ooid": "1237", "Product": "Fennicky", "Version": "1.0" }), }) def get_raw_crash(self, ooid): return self.store[ooid] def get_raw_dump(self, ooid): return "this is a fake dump" def new_ooids(self): for k in self.store.keys(): yield k config = DotDict({ "logger": MagicMock(), "number_of_threads": 2, "maximum_queue_size": 2, "queue": DotDict({"crashqueue_class": None}), "source": DotDict({"crashstorage_class": FakeStorageSource}), "destination": DotDict({"crashstorage_class": None}), "producer_consumer": DotDict({ "producer_consumer_class": ThreadedTaskManager, "logger": MagicMock(), "number_of_threads": 1, "maximum_queue_size": 1, }), }) fts_app = FetchTransformSaveApp(config) with pytest.raises(TypeError): fts_app.main()
def test_queue_iterator(self): faked_finished_func = Mock() class FakeQueue(object): def __init__(self): self.first = True def new_crashes(self): if self.first: # make the iterator act as if exhausted on the very # first try self.first = False else: for k in range(999): # ensure that both forms (a single value or the # (args, kwargs) form are accepted.) if k % 4: yield k else: yield ((k, ), { "finished_func": faked_finished_func }) for k in range(2): yield None class FakeStorageDestination(object): def __init__(self, config): self.store = DotDict() self.dumps = DotDict() def save_raw_crash(self, raw_crash, dump, crash_id): self.store[crash_id] = raw_crash self.dumps[crash_id] = dump config = DotDict({ "logger": MagicMock(), "number_of_threads": 2, "maximum_queue_size": 2, "queue": DotDict({"crashqueue_class": FakeQueue}), "source": DotDict({"crashstorage_class": None}), "destination": DotDict({"crashstorage_class": FakeStorageDestination}), "producer_consumer": DotDict({ "producer_consumer_class": ThreadedTaskManager, "logger": MagicMock(), "number_of_threads": 1, "maximum_queue_size": 1, }), }) fts_app = FetchTransformSaveApp(config) fts_app.queue = FakeQueue() fts_app.source = None fts_app.destination = FakeStorageDestination error_detected = False no_finished_function_counter = 0 for x, y in zip(range(1002), (a for a in fts_app.source_iterator())): if x == 0: # the iterator is exhausted on the 1st try and should have # yielded a None before starting over assert y is None elif x < 1000: if x - 1 != y[0][0] and not error_detected: error_detected = True assert x == y, "iterator fails on iteration %d: %s" % (x, y) # invoke that finished func to ensure that we've got the # right object try: y[1]["finished_func"]() except KeyError: no_finished_function_counter += 1 else: if y is not None and not error_detected: error_detected = True assert x is None, "iterator fails on iteration %d: %s" % ( x, y) assert faked_finished_func.call_count == (999 - no_finished_function_counter)