def __init__(self, config, quit_check_callback):
self.store = DotDict({
'1234':
DotDict({
'ooid': '1234',
'Product': 'FireSquid',
'Version': '1.0'
}),
'1235':
DotDict({
'ooid': '1235',
'Product': 'ThunderRat',
'Version': '1.0'
}),
'1236':
DotDict({
'ooid': '1236',
'Product': 'Caminimal',
'Version': '1.0'
}),
'1237':
DotDict({
'ooid': '1237',
'Product': 'Fennicky',
'Version': '1.0'
}),
})
self.number_of_close_calls = 0
def test_save_raw_crash_normal(self):
config = self._setup_config()
crash_store = RabbitMQCrashStorage(config)
# test for "legacy_processing" missing from crash
crash_store.save_raw_crash(raw_crash=DotDict(),
dumps=DotDict(),
crash_id='crash_id')
ok_(not crash_store.transaction.called)
config.logger.reset_mock()
# test for normal save
raw_crash = DotDict()
raw_crash.legacy_processing = 0
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
crash_store.transaction.assert_called_with(
crash_store._save_raw_crash_transaction, 'crash_id')
crash_store.transaction.reset_mock()
# test for save rejection because of "legacy_processing"
raw_crash = DotDict()
raw_crash.legacy_processing = 5
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
ok_(not crash_store.transaction.called)
def test_save_raw_crash_no_legacy(self):
config = self._setup_config()
config.filter_on_legacy_processing = False
crash_store = RabbitMQCrashStorage(config)
# test for "legacy_processing" missing from crash
crash_store.save_raw_crash(raw_crash=DotDict(),
dumps=DotDict(),
crash_id='crash_id')
crash_store.transaction.assert_called_with(
crash_store._save_raw_crash_transaction, 'crash_id')
config.logger.reset_mock()
# test for normal save
raw_crash = DotDict()
raw_crash.legacy_processing = 0
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
crash_store.transaction.assert_called_with(
crash_store._save_raw_crash_transaction, 'crash_id')
crash_store.transaction.reset_mock()
# test for save without regard to "legacy_processing" value
raw_crash = DotDict()
raw_crash.legacy_processing = 5
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
crash_store.transaction.assert_called_with(
crash_store._save_raw_crash_transaction, 'crash_id')
def test_bogus_source_iter_and_worker(self):
class TestFTSAppClass(FetchTransformSaveApp):
def __init__(self, config):
super(TestFTSAppClass, self).__init__(config)
self.the_list = []
def _setup_source_and_destination(self):
self.source = Mock()
self.destination = Mock()
pass
def _create_iter(self):
for x in xrange(5):
yield ((x, ), {})
def transform(self, anItem):
self.the_list.append(anItem)
logger = SilentFakeLogger()
config = DotDict({
'logger':
logger,
'number_of_threads':
2,
'maximum_queue_size':
2,
'number_of_submissions':
'all',
'source':
DotDict({'crashstorage_class': None}),
'destination':
DotDict({'crashstorage_class': None}),
'producer_consumer':
DotDict({
'producer_consumer_class': TaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = TestFTSAppClass(config)
fts_app.main()
ok_(
len(fts_app.the_list) == 5, 'expected to do 5 inserts, '
'but %d were done instead' % len(fts_app.the_list))
ok_(
sorted(fts_app.the_list) == range(5),
'expected %s, but got %s' % (range(5), sorted(fts_app.the_list)))
def test_constructor(self):
faked_connection_object = Mock()
config = DotDict()
conn = Connection(
config,
faked_connection_object
)
ok_(conn.config is config)
ok_(conn.connection is faked_connection_object)
faked_connection_object.channel.called_once_with()
eq_(
faked_connection_object.channel.return_value
.queue_declare.call_count,
3
)
expected_queue_declare_call_args = [
call(queue='socorro.normal', durable=True),
call(queue='socorro.priority', durable=True),
call(queue='socorro.reprocessing', durable=True),
]
eq_(
faked_connection_object.channel.return_value.queue_declare.call_args_list,
expected_queue_declare_call_args
)
def setup_mocked_s3_storage(
self,
executor=TransactionExecutor,
executor_for_gets=TransactionExecutor,
storage_class='BotoS3CrashStorage',
host='',
port=0,
resource_class=S3ConnectionContext,
**extra
):
config = DotDict({
'resource_class': resource_class,
'logger': mock.Mock(),
'host': host,
'port': port,
'access_key': 'this is the access key',
'secret_access_key': 'secrets',
'bucket_name': 'silliness',
'keybuilder_class': KeyBuilderBase,
'prefix': 'dev',
'calling_format': mock.Mock()
})
config.update(extra)
s3_conn = resource_class(config)
s3_conn._connect_to_endpoint = mock.Mock()
s3_conn._mocked_connection = s3_conn._connect_to_endpoint.return_value
s3_conn._calling_format.return_value = mock.Mock()
s3_conn._CreateError = mock.Mock()
s3_conn.ResponseError = mock.Mock()
s3_conn._open = mock.MagicMock()
return s3_conn
def test_doing_work_with_two_workers_and_generator(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 2
config.maximum_queue_size = 2
my_list = []
def insert_into_list(anItem):
my_list.append(anItem)
ttm = ThreadedTaskManager(config,
task_func=insert_into_list,
job_source_iterator=(((x,), {}) for x in
xrange(10))
)
try:
ttm.start()
time.sleep(0.2)
ok_(len(ttm.thread_list) == 2,
"expected 2 threads, but found %d"
% len(ttm.thread_list))
ok_(len(my_list) == 10,
'expected to do 10 inserts, '
'but %d were done instead' % len(my_list))
ok_(sorted(my_list) == range(10),
'expected %s, but got %s' % (range(10),
sorted(my_list)))
except Exception:
# we got threads to join
ttm.wait_for_completion()
raise
def test_doing_work_with_one_worker(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 1
config.maximum_queue_size = 1
my_list = []
def insert_into_list(anItem):
my_list.append(anItem)
ttm = ThreadedTaskManager(config,
task_func=insert_into_list
)
try:
ttm.start()
time.sleep(0.2)
ok_(len(my_list) == 10,
'expected to do 10 inserts, '
'but %d were done instead' % len(my_list))
ok_(my_list == range(10),
'expected %s, but got %s' % (range(10), my_list))
ttm.stop()
except Exception:
# we got threads to join
ttm.wait_for_completion()
raise
def test_get_iterator(self):
config = DotDict()
config.logger = self.logger
config.quit_on_empty_queue = False
tm = TaskManager(
config,
job_source_iterator=range(1),
)
eq_(tm._get_iterator(), [0])
def an_iter(self):
for i in range(5):
yield i
tm = TaskManager(
config,
job_source_iterator=an_iter,
)
eq_([x for x in tm._get_iterator()], [0, 1, 2, 3, 4])
class X(object):
def __init__(self, config):
self.config = config
def __iter__(self):
for key in self.config:
yield key
tm = TaskManager(config, job_source_iterator=X(config))
eq_([x for x in tm._get_iterator()], [y for y in config.keys()])
def test_blocking_start(self):
config = DotDict()
config.logger = self.logger
config.idle_delay = 1
config.quit_on_empty_queue = False
class MyTaskManager(TaskManager):
def _responsive_sleep(self,
seconds,
wait_log_interval=0,
wait_reason=''):
try:
if self.count >= 2:
self.quit = True
self.count += 1
except AttributeError:
self.count = 0
tm = MyTaskManager(config, task_func=Mock())
waiting_func = Mock()
tm.blocking_start(waiting_func=waiting_func)
eq_(tm.task_func.call_count, 10)
eq_(waiting_func.call_count, 0)
def test_executor_identity(self):
config = DotDict()
config.logger = self.logger
tm = TaskManager(
config,
job_source_iterator=range(1),
)
tm._pid = 666
eq_(tm.executor_identity(), '666-MainThread')
def test_close(self):
faked_connection_object = Mock()
config = DotDict()
conn = Connection(
config,
faked_connection_object
)
conn.close()
faked_connection_object.close.assert_called_once_with()
def test_constuctor1(self):
config = DotDict()
config.logger = self.logger
config.quit_on_empty_queue = False
tm = TaskManager(config)
ok_(tm.config == config)
ok_(tm.logger == self.logger)
ok_(tm.task_func == default_task_func)
ok_(tm.quit is False)
def test_transaction_ack_crash(self):
config = self._setup_config()
connection = Mock()
ack_token = DotDict()
ack_token.delivery_tag = 1
crash_id = 'some-crash-id'
crash_store = RabbitMQCrashStorage(config)
crash_store._transaction_ack_crash(connection, crash_id, ack_token)
connection.channel.basic_ack.assert_called_once_with(delivery_tag=1)
def _get_raw_crash_from_form(self):
"""this method creates the raw_crash and the dumps mapping using the
POST form"""
dumps = MemoryDumpsMapping()
raw_crash = DotDict()
raw_crash.dump_checksums = DotDict()
for name, value in self._form_as_mapping().iteritems():
name = self._no_x00_character(name)
if isinstance(value, basestring):
if name != "dump_checksums":
raw_crash[name] = self._no_x00_character(value)
elif hasattr(value, 'file') and hasattr(value, 'value'):
dumps[name] = value.value
raw_crash.dump_checksums[name] = \
self.checksum_method(value.value).hexdigest()
elif isinstance(value, int):
raw_crash[name] = value
else:
raw_crash[name] = value.value
return raw_crash, dumps
def _setup_config(self):
config = DotDict()
config.transaction_executor_class = Mock()
config.backoff_delays = (0, 0, 0)
config.logger = Mock()
config.rabbitmq_class = MagicMock()
config.routing_key = 'socorro.normal'
config.filter_on_legacy_processing = True
config.redactor_class = Redactor
config.forbidden_keys = Redactor.required_config.forbidden_keys.default
config.throttle = 100
return config
def test_constuctor1(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 1
config.maximum_queue_size = 1
ttm = ThreadedTaskManager(config)
try:
ok_(ttm.config == config)
ok_(ttm.logger == self.logger)
ok_(ttm.task_func == default_task_func)
ok_(ttm.quit is False)
finally:
# we got threads to join
ttm._kill_worker_threads()
def test_blocking_start_with_quit_on_empty(self):
config = DotDict()
config.logger = self.logger
config.idle_delay = 1
config.quit_on_empty_queue = True
tm = TaskManager(config, task_func=Mock())
waiting_func = Mock()
tm.blocking_start(waiting_func=waiting_func)
eq_(tm.task_func.call_count, 10)
eq_(waiting_func.call_count, 0)
def test_no_source(self):
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
logger = SilentFakeLogger()
config = DotDict({
'logger':
logger,
'number_of_threads':
2,
'maximum_queue_size':
2,
'number_of_submissions':
'forever',
'source':
DotDict({'crashstorage_class': None}),
'destination':
DotDict({'crashstorage_class': FakeStorageDestination}),
'producer_consumer':
DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
assert_raises(TypeError, fts_app.main)
def _setup_config(self):
config = DotDict()
config.host = 'localhost'
config.virtual_host = '/'
config.port = '5672'
config.rabbitmq_user = '******'
config.rabbitmq_password = '******'
config.standard_queue_name = 'dwight'
config.priority_queue_name = 'wilma'
config.reprocessing_queue_name = 'betty'
config.rabbitmq_connection_wrapper_class = Connection
config.executor_identity = lambda: 'MainThread'
return config
def test_new_crash_duplicate_discovered(self):
""" Tests queue with standard queue items only
"""
config = self._setup_config()
config.transaction_executor_class = TransactionExecutor
crash_store = RabbitMQCrashStorage(config)
crash_store.rabbitmq.config.standard_queue_name = 'socorro.normal'
crash_store.rabbitmq.config.reprocessing_queue_name = \
'socorro.reprocessing'
crash_store.rabbitmq.config.priority_queue_name = 'socorro.priority'
faked_methodframe = DotDict()
faked_methodframe.delivery_tag = 'delivery_tag'
test_queue = [
(None, None, None),
(faked_methodframe, '1', 'normal_crash_id'),
(None, None, None),
]
def basic_get(queue='socorro.priority'):
if len(test_queue) == 0:
raise StopIteration
return test_queue.pop()
crash_store.rabbitmq.return_value.__enter__.return_value \
.channel.basic_get = MagicMock(side_effect=basic_get)
transaction_connection = crash_store.transaction.db_conn_context_source \
.return_value.__enter__.return_value
# load the cache as if this crash had alredy been seen
crash_store.acknowledgement_token_cache['normal_crash_id'] = \
faked_methodframe
for result in crash_store.new_crashes():
# new crash should be suppressed
eq_(None, result)
# we should ack the new crash even though we did use it for processing
transaction_connection.channel.basic_ack \
.assert_called_with(
delivery_tag=faked_methodframe.delivery_tag
)
def test_blocking_start_with_quit_on_empty(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 2
config.maximum_queue_size = 2
config.quit_on_empty_queue = True
tm = ThreadedTaskManager(
config,
task_func=Mock()
)
waiting_func = Mock()
tm.blocking_start(waiting_func=waiting_func)
eq_(
tm.task_func.call_count,
10
)
def test_start1(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 1
config.maximum_queue_size = 1
ttm = ThreadedTaskManager(config)
try:
ttm.start()
time.sleep(0.2)
ok_(ttm.queuing_thread.isAlive(),
"the queing thread is not running")
ok_(len(ttm.thread_list) == 1,
"where's the worker thread?")
ok_(ttm.thread_list[0].isAlive(),
"the worker thread is stillborn")
ttm.stop()
ok_(ttm.queuing_thread.isAlive() is False,
"the queuing thread did not stop")
except Exception:
# we got threads to join
ttm.wait_for_completion()
def test_task_raises_unexpected_exception(self):
global count
count = 0
def new_iter():
for x in xrange(10):
yield (x,)
my_list = []
def insert_into_list(anItem):
global count
count += 1
if count == 4:
raise Exception('Unexpected')
my_list.append(anItem)
config = DotDict()
config.logger = self.logger
config.number_of_threads = 1
config.maximum_queue_size = 1
config.job_source_iterator = new_iter
config.task_func = insert_into_list
ttm = ThreadedTaskManagerWithConfigSetup(config)
try:
ttm.start()
time.sleep(0.2)
ok_(len(ttm.thread_list) == 1,
"expected 1 threads, but found %d"
% len(ttm.thread_list))
ok_(sorted(my_list) == [0, 1, 2, 4, 5, 6, 7, 8, 9],
'expected %s, but got %s'
% ([0, 1, 2, 5, 6, 7, 8, 9], sorted(my_list)))
ok_(len(my_list) == 9,
'expected to do 9 inserts, '
'but %d were done instead' % len(my_list))
except Exception:
# we got threads to join
ttm.wait_for_completion()
raise
def test_save_raw_crash_normal_throttle(self, randint_mock):
random_ints = [100, 49, 50, 51, 1, 100]
def side_effect(*args, **kwargs):
return random_ints.pop(0)
randint_mock.side_effect = side_effect
config = self._setup_config()
config.throttle = 50
crash_store = RabbitMQCrashStorage(config)
# test for "legacy_processing" missing from crash #0: 100
crash_store.save_raw_crash(raw_crash=DotDict(),
dumps=DotDict(),
crash_id='crash_id')
ok_(not crash_store.transaction.called)
config.logger.reset_mock()
# test for normal save #1: 49
raw_crash = DotDict()
raw_crash.legacy_processing = 0
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
crash_store.transaction.assert_called_with(
crash_store._save_raw_crash_transaction, 'crash_id')
crash_store.transaction.reset_mock()
# test for normal save #2: 50
raw_crash = DotDict()
raw_crash.legacy_processing = 0
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
crash_store.transaction.assert_called_with(
crash_store._save_raw_crash_transaction, 'crash_id')
crash_store.transaction.reset_mock()
# test for normal save #3: 51
raw_crash = DotDict()
raw_crash.legacy_processing = 0
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
ok_(not crash_store.transaction.called)
crash_store.transaction.reset_mock()
# test for save rejection because of "legacy_processing" #4: 1
raw_crash = DotDict()
raw_crash.legacy_processing = 5
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
ok_(not crash_store.transaction.called)
# test for save rejection because of "legacy_processing" #5: 100
raw_crash = DotDict()
raw_crash.legacy_processing = 5
crash_store.save_raw_crash(raw_crash=raw_crash,
dumps=DotDict,
crash_id='crash_id')
ok_(not crash_store.transaction.called)
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
self.number_of_close_calls = 0
def test_source_iterator(self):
faked_finished_func = Mock()
class FakeStorageSource(object):
def __init__(self):
self.first = True
def new_crashes(self):
if self.first:
# make the iterator act as if exhausted on the very
# first try
self.first = False
else:
for k in range(999):
# ensure that both forms (a single value or the
# (args, kwargs) form are accepted.)
if k % 4:
yield k
else:
yield ((k, ), {
"finished_func": faked_finished_func
})
for k in range(2):
yield None
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
logger = SilentFakeLogger()
config = DotDict({
'logger':
logger,
'number_of_threads':
2,
'maximum_queue_size':
2,
'number_of_submissions':
'forever',
'source':
DotDict({'crashstorage_class': FakeStorageSource}),
'destination':
DotDict({'crashstorage_class': FakeStorageDestination}),
'producer_consumer':
DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
fts_app.source = FakeStorageSource()
fts_app.destination = FakeStorageDestination
error_detected = False
no_finished_function_counter = 0
for x, y in zip(xrange(1002), (a for a in fts_app.source_iterator())):
if x == 0:
# the iterator is exhausted on the 1st try and should have
# yielded a None before starting over
ok_(y is None)
elif x < 1000:
if x - 1 != y[0][0] and not error_detected:
error_detected = True
eq_(x, y, 'iterator fails on iteration %d: %s' % (x, y))
# invoke that finished func to ensure that we've got the
# right object
try:
y[1]['finished_func']()
except KeyError:
no_finished_function_counter += 1
else:
if y is not None and not error_detected:
error_detected = True
ok_(x is None,
'iterator fails on iteration %d: %s' % (x, y))
eq_(faked_finished_func.call_count, 999 - no_finished_function_counter)
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
def test_no_destination(self):
class FakeStorageSource(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict({
'1234':
DotDict({
'ooid': '1234',
'Product': 'FireSquid',
'Version': '1.0'
}),
'1235':
DotDict({
'ooid': '1235',
'Product': 'ThunderRat',
'Version': '1.0'
}),
'1236':
DotDict({
'ooid': '1236',
'Product': 'Caminimal',
'Version': '1.0'
}),
'1237':
DotDict({
'ooid': '1237',
'Product': 'Fennicky',
'Version': '1.0'
}),
})
def get_raw_crash(self, ooid):
return self.store[ooid]
def get_raw_dump(self, ooid):
return 'this is a fake dump'
def new_ooids(self):
for k in self.store.keys():
yield k
logger = SilentFakeLogger()
config = DotDict({
'logger':
logger,
'number_of_threads':
2,
'maximum_queue_size':
2,
'number_of_submissions':
'forever',
'source':
DotDict({'crashstorage_class': FakeStorageSource}),
'destination':
DotDict({'crashstorage_class': None}),
'producer_consumer':
DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = FetchTransformSaveApp(config)
assert_raises(TypeError, fts_app.main)
def test_bogus_source_and_destination(self):
class NonInfiniteFTSAppClass(FetchTransformSaveApp):
def _basic_iterator(self):
for x in self.source.new_crashes():
yield ((x, ), {})
class FakeStorageSource(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict({
'1234':
DotDict({
'ooid': '1234',
'Product': 'FireSquid',
'Version': '1.0'
}),
'1235':
DotDict({
'ooid': '1235',
'Product': 'ThunderRat',
'Version': '1.0'
}),
'1236':
DotDict({
'ooid': '1236',
'Product': 'Caminimal',
'Version': '1.0'
}),
'1237':
DotDict({
'ooid': '1237',
'Product': 'Fennicky',
'Version': '1.0'
}),
})
self.number_of_close_calls = 0
def get_raw_crash(self, ooid):
return self.store[ooid]
def get_raw_dumps(self, ooid):
return {'upload_file_minidump': 'this is a fake dump'}
def new_crashes(self):
for k in self.store.keys():
yield k
def close(self):
self.number_of_close_calls += 1
class FakeStorageDestination(object):
def __init__(self, config, quit_check_callback):
self.store = DotDict()
self.dumps = DotDict()
self.number_of_close_calls = 0
def save_raw_crash(self, raw_crash, dump, crash_id):
self.store[crash_id] = raw_crash
self.dumps[crash_id] = dump
def close(self):
self.number_of_close_calls += 1
logger = SilentFakeLogger()
config = DotDict({
'logger':
logger,
'number_of_threads':
2,
'maximum_queue_size':
2,
'number_of_submissions':
'all',
'source':
DotDict({'crashstorage_class': FakeStorageSource}),
'destination':
DotDict({'crashstorage_class': FakeStorageDestination}),
'producer_consumer':
DotDict({
'producer_consumer_class': ThreadedTaskManager,
'logger': logger,
'number_of_threads': 1,
'maximum_queue_size': 1
})
})
fts_app = NonInfiniteFTSAppClass(config)
fts_app.main()
source = fts_app.source
destination = fts_app.destination
eq_(source.store, destination.store)
eq_(len(destination.dumps), 4)
eq_(destination.dumps['1237'], source.get_raw_dumps('1237'))
# ensure that each storage system had its close called
eq_(source.number_of_close_calls, 1)
eq_(destination.number_of_close_calls, 1)
