def test_no_source(self):
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
logger = get_logger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'number_of_submissions': 'forever',
'source': DotDict({'crashstorage_class': None}),
'destination': DotDict({'crashstorage_class': FakeStorageDestination}),
'producer_consumer': DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
with pytest.raises(TypeError):
fts_app.main()
def test_source_iterator(self):
class FakeStorageSource(object):
def __init__(self):
self.first = True
def new_crashes(self):
if self.first:
self.first = False
else:
for k in range(999):
yield k
for k in range(2):
yield None
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
logger = SilentFakeLogger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage_class':
FakeStorageSource}),
'destination': DotDict({'crashstorage_class':
FakeStorageDestination}),
'producer_consumer': DotDict({'producer_consumer_class':
ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1}
)
})
fts_app = FetchTransformSaveApp(config)
fts_app.source = FakeStorageSource()
fts_app.destination = FakeStorageDestination
error_detected = False
for x, y in zip(xrange(1002), (a for a in fts_app.source_iterator())):
if x == 0:
self.assertTrue(y is None)
elif x < 1000:
if x - 1 != y[0][0] and not error_detected:
error_detected = True
self.assertEqual(x, y,
'iterator fails on iteration %d' % x)
else:
print x, y
if y is not None and not error_detected:
error_detected = True
self.assertTrue(x is None,
'iterator fails on iteration %d' % x)
def test_no_source(self):
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
config = DotDict({
'logger': MagicMock(),
'number_of_threads': 2,
'maximum_queue_size': 2,
'number_of_submissions': 'forever',
'source': DotDict({'crashstorage_class': None}),
'destination': DotDict({'crashstorage_class': FakeStorageDestination}),
'producer_consumer': DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': MagicMock(),
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
with pytest.raises(TypeError):
fts_app.main()
def test_source_iterator(self):
class FakeStorageSource(object):
def __init__(self):
self.first = True
def new_crashes(self):
if self.first:
self.first = False
else:
for k in range(999):
yield k
for k in range(2):
yield None
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
logger = SilentFakeLogger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage_class':
FakeStorageSource}),
'destination': DotDict({'crashstorage_class':
FakeStorageDestination}),
'producer_consumer': DotDict({'producer_consumer_class':
ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1}
)
})
fts_app = FetchTransformSaveApp(config)
fts_app.source = FakeStorageSource()
fts_app.destination = FakeStorageDestination
error_detected = False
for x, y in zip(xrange(1002), (a for a in fts_app.source_iterator())):
if x == 0:
self.assertTrue(y is None)
elif x < 1000:
if x - 1 != y[0][0] and not error_detected:
error_detected = True
self.assertEqual(x, y,
'iterator fails on iteration %d' % x)
else:
if y is not None and not error_detected:
error_detected = True
self.assertTrue(x is None,
'iterator fails on iteration %d' % x)
def test_no_destination(self):
class FakeStorageSource(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict({
'1234': DotDict({
'ooid': '1234',
'Product': 'FireSquid',
'Version': '1.0'
}),
'1235': DotDict({
'ooid': '1235',
'Product': 'ThunderRat',
'Version': '1.0'
}),
'1236': DotDict({
'ooid': '1236',
'Product': 'Caminimal',
'Version': '1.0'
}),
'1237': DotDict({
'ooid': '1237',
'Product': 'Fennicky',
'Version': '1.0'
}),
})
def get_raw_crash(self, ooid):
return self.store[ooid]
def get_raw_dump(self, ooid):
return 'this is a fake dump'
def new_ooids(self):
for k in self.store.keys():
yield k
logger = get_logger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'number_of_submissions': 'forever',
'source': DotDict({'crashstorage_class': FakeStorageSource}),
'destination': DotDict({'crashstorage_class': None}),
'producer_consumer': DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
with pytest.raises(TypeError):
fts_app.main()
def test_no_destination(self):
class FakeStorageSource(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict({
'1234': DotDict({
'ooid': '1234',
'Product': 'FireSquid',
'Version': '1.0'
}),
'1235': DotDict({
'ooid': '1235',
'Product': 'ThunderRat',
'Version': '1.0'
}),
'1236': DotDict({
'ooid': '1236',
'Product': 'Caminimal',
'Version': '1.0'
}),
'1237': DotDict({
'ooid': '1237',
'Product': 'Fennicky',
'Version': '1.0'
}),
})
def get_raw_crash(self, ooid):
return self.store[ooid]
def get_raw_dump(self, ooid):
return 'this is a fake dump'
def new_ooids(self):
for k in self.store.keys():
yield k
config = DotDict({
'logger': MagicMock(),
'number_of_threads': 2,
'maximum_queue_size': 2,
'number_of_submissions': 'forever',
'source': DotDict({'crashstorage_class': FakeStorageSource}),
'destination': DotDict({'crashstorage_class': None}),
'producer_consumer': DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': MagicMock(),
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
with pytest.raises(TypeError):
fts_app.main()
def test_no_source(self):
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump):
self.store[raw_crash.ooid] = raw_crash
self.dumps[raw_crash.ooid] = dump
logger = SilentFakeLogger()
config = DotDict({
'logger':
logger,
'number_of_threads':
2,
'maximum_queue_size':
2,
'source':
DotDict({'crashstorage': None}),
'destination':
DotDict({'crashstorage': FakeStorageDestination})
})
fts_app = FetchTransformSaveApp(config)
self.assertRaises(TypeError, fts_app.main)
def test_no_destination(self):
class FakeStorageSource(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict({
'1234':
DotDict({
'ooid': '1234',
'Product': 'FireFloozy',
'Version': '1.0'
}),
'1235':
DotDict({
'ooid': '1235',
'Product': 'ThunderRat',
'Version': '1.0'
}),
'1236':
DotDict({
'ooid': '1236',
'Product': 'Caminimal',
'Version': '1.0'
}),
'1237':
DotDict({
'ooid': '1237',
'Product': 'Fennicky',
'Version': '1.0'
}),
})
def get_raw_crash(self, ooid):
return self.store[ooid]
def get_dump(self, ooid):
return 'this is a fake dump'
def new_ooids(self):
for k in self.store.keys():
yield k
logger = SilentFakeLogger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'source': DotDict({'crashstorage': FakeStorageSource}),
'destination': DotDict({'crashstorage': None})
})
fts_app = FetchTransformSaveApp(config)
self.assertRaises(TypeError, fts_app.main)
def test_no_source(self):
class FakeStorageDestination(object):
def __init__(self, config):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
config = DotDict({
"logger":
MagicMock(),
"number_of_threads":
2,
"maximum_queue_size":
2,
"queue":
DotDict({"crashqueue_class": None}),
"source":
DotDict({"crashstorage_class": None}),
"destination":
DotDict({"crashstorage_class": FakeStorageDestination}),
"producer_consumer":
DotDict({
"producer_consumer_class": ThreadedTaskManager,
"logger": MagicMock(),
"number_of_threads": 1,
"maximum_queue_size": 1,
}),
})
fts_app = FetchTransformSaveApp(config)
with pytest.raises(TypeError):
fts_app.main()
def test_source_iterator(self):
faked_finished_func = Mock()
class FakeStorageSource(object):
def __init__(self):
self.first = True
def new_crashes(self):
if self.first:
# make the iterator act as if exhausted on the very
# first try
self.first = False
else:
for k in range(999):
# ensure that both forms (a single value or the
# (args, kwargs) form are accepted.)
if k % 4:
yield k
else:
yield (
(k, ),
{"finished_func": faked_finished_func}
)
for k in range(2):
yield None
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
logger = get_logger()
config = DotDict({
'logger': logger,
'number_of_threads': 2,
'maximum_queue_size': 2,
'number_of_submissions': 'forever',
'source': DotDict({'crashstorage_class': FakeStorageSource}),
'destination': DotDict({'crashstorage_class': FakeStorageDestination}),
'producer_consumer': DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
fts_app.source = FakeStorageSource()
fts_app.destination = FakeStorageDestination
error_detected = False
no_finished_function_counter = 0
for x, y in zip(range(1002), (a for a in fts_app.source_iterator())):
if x == 0:
# the iterator is exhausted on the 1st try and should have
# yielded a None before starting over
assert y is None
elif x < 1000:
if x - 1 != y[0][0] and not error_detected:
error_detected = True
assert x == y, 'iterator fails on iteration %d: %s' % (x, y)
# invoke that finished func to ensure that we've got the
# right object
try:
y[1]['finished_func']()
except KeyError:
no_finished_function_counter += 1
else:
if y is not None and not error_detected:
error_detected = True
assert x is None, 'iterator fails on iteration %d: %s' % (x, y)
assert faked_finished_func.call_count == (999 - no_finished_function_counter)
def test_source_iterator(self):
faked_finished_func = Mock()
class FakeStorageSource(object):
def __init__(self):
self.first = True
def new_crashes(self):
if self.first:
# make the iterator act as if exhausted on the very
# first try
self.first = False
else:
for k in range(999):
# ensure that both forms (a single value or the
# (args, kwargs) form are accepted.)
if k % 4:
yield k
else:
yield ((k, ), {
"finished_func": faked_finished_func
})
for k in range(2):
yield None
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
logger = SilentFakeLogger()
config = DotDict({
'logger':
logger,
'number_of_threads':
2,
'maximum_queue_size':
2,
'number_of_submissions':
'forever',
'source':
DotDict({'crashstorage_class': FakeStorageSource}),
'destination':
DotDict({'crashstorage_class': FakeStorageDestination}),
'producer_consumer':
DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
fts_app.source = FakeStorageSource()
fts_app.destination = FakeStorageDestination
error_detected = False
no_finished_function_counter = 0
for x, y in zip(range(1002), (a for a in fts_app.source_iterator())):
if x == 0:
# the iterator is exhausted on the 1st try and should have
# yielded a None before starting over
assert y is None
elif x < 1000:
if x - 1 != y[0][0] and not error_detected:
error_detected = True
assert x == y, 'iterator fails on iteration %d: %s' % (x,
y)
# invoke that finished func to ensure that we've got the
# right object
try:
y[1]['finished_func']()
except KeyError:
no_finished_function_counter += 1
else:
if y is not None and not error_detected:
error_detected = True
assert x is None, 'iterator fails on iteration %d: %s' % (
x, y)
assert faked_finished_func.call_count == (999 -
no_finished_function_counter)
def test_no_destination(self):
class FakeStorageSource(object):
def __init__(self, config):
self.store = DotDict({
"1234":
DotDict({
"ooid": "1234",
"Product": "FireSquid",
"Version": "1.0"
}),
"1235":
DotDict({
"ooid": "1235",
"Product": "ThunderRat",
"Version": "1.0"
}),
"1236":
DotDict({
"ooid": "1236",
"Product": "Caminimal",
"Version": "1.0"
}),
"1237":
DotDict({
"ooid": "1237",
"Product": "Fennicky",
"Version": "1.0"
}),
})
def get_raw_crash(self, ooid):
return self.store[ooid]
def get_raw_dump(self, ooid):
return "this is a fake dump"
def new_ooids(self):
for k in self.store.keys():
yield k
config = DotDict({
"logger":
MagicMock(),
"number_of_threads":
2,
"maximum_queue_size":
2,
"queue":
DotDict({"crashqueue_class": None}),
"source":
DotDict({"crashstorage_class": FakeStorageSource}),
"destination":
DotDict({"crashstorage_class": None}),
"producer_consumer":
DotDict({
"producer_consumer_class": ThreadedTaskManager,
"logger": MagicMock(),
"number_of_threads": 1,
"maximum_queue_size": 1,
}),
})
fts_app = FetchTransformSaveApp(config)
with pytest.raises(TypeError):
fts_app.main()
def test_queue_iterator(self):
faked_finished_func = Mock()
class FakeQueue(object):
def __init__(self):
self.first = True
def new_crashes(self):
if self.first:
# make the iterator act as if exhausted on the very
# first try
self.first = False
else:
for k in range(999):
# ensure that both forms (a single value or the
# (args, kwargs) form are accepted.)
if k % 4:
yield k
else:
yield ((k, ), {
"finished_func": faked_finished_func
})
for k in range(2):
yield None
class FakeStorageDestination(object):
def __init__(self, config):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
config = DotDict({
"logger":
MagicMock(),
"number_of_threads":
2,
"maximum_queue_size":
2,
"queue":
DotDict({"crashqueue_class": FakeQueue}),
"source":
DotDict({"crashstorage_class": None}),
"destination":
DotDict({"crashstorage_class": FakeStorageDestination}),
"producer_consumer":
DotDict({
"producer_consumer_class": ThreadedTaskManager,
"logger": MagicMock(),
"number_of_threads": 1,
"maximum_queue_size": 1,
}),
})
fts_app = FetchTransformSaveApp(config)
fts_app.queue = FakeQueue()
fts_app.source = None
fts_app.destination = FakeStorageDestination
error_detected = False
no_finished_function_counter = 0
for x, y in zip(range(1002), (a for a in fts_app.source_iterator())):
if x == 0:
# the iterator is exhausted on the 1st try and should have
# yielded a None before starting over
assert y is None
elif x < 1000:
if x - 1 != y[0][0] and not error_detected:
error_detected = True
assert x == y, "iterator fails on iteration %d: %s" % (x,
y)
# invoke that finished func to ensure that we've got the
# right object
try:
y[1]["finished_func"]()
except KeyError:
no_finished_function_counter += 1
else:
if y is not None and not error_detected:
error_detected = True
assert x is None, "iterator fails on iteration %d: %s" % (
x, y)
assert faked_finished_func.call_count == (999 -
no_finished_function_counter)
