def test_basic_postgres_save_processed_success_3_truncations(self):
config = DotDict()
config.database_class = mock.MagicMock()
config.transaction_executor_class = TransactionExecutorWithInfiniteBackoff
config.redactor_class = mock.Mock()
config.backoff_delays = [1]
config.wait_log_interval = 10
config.logger = mock.Mock()
mocked_database_connection_source = config.database_class.return_value
mocked_connection = (
mocked_database_connection_source.return_value
.__enter__.return_value
)
mocked_cursor = mocked_connection.cursor.return_value.__enter__.return_value
# the call to be tested
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.save_processed(a_processed_crash_with_everything_too_long)
assert mocked_database_connection_source.call_count == 1
assert mocked_cursor.execute.call_count == 3
# check correct fragments
first_call = mocked_cursor.execute.call_args_list[0]
assert first_call[0][1] == a_processed_report_with_everything_truncated * 2
def test_basic_postgres_save_processed_success(self):
config = DotDict()
config.database_class = mock.MagicMock()
config.transaction_executor_class = TransactionExecutorWithInfiniteBackoff
config.redactor_class = mock.Mock()
config.backoff_delays = [1]
config.wait_log_interval = 10
config.logger = mock.Mock()
mocked_database_connection_source = config.database_class.return_value
mocked_connection = (mocked_database_connection_source.return_value.
__enter__.return_value)
mocked_cursor = mocked_connection.cursor.return_value.__enter__.return_value
# the call to be tested
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.save_processed(a_processed_crash)
eq_(mocked_database_connection_source.call_count, 1)
eq_(mocked_cursor.execute.call_count, 5)
# check correct fragments
sql_fragments = [
"UPDATE reports_20120402", 'select id from plugins',
'delete from plugins_reports_20120402',
'insert into plugins_reports_20120402',
'UPDATE processed_crashes_20120402'
]
for a_call, a_fragment in zip(mocked_cursor.execute.call_args_list,
sql_fragments):
ok_(a_fragment in a_call[0][0])
def test_basic_postgres_save_processed_success_3_truncations(self):
config = DotDict()
config.database_class = mock.MagicMock()
config.transaction_executor_class = TransactionExecutorWithInfiniteBackoff
config.redactor_class = mock.Mock()
config.backoff_delays = [1]
config.wait_log_interval = 10
config.logger = mock.Mock()
mocked_database_connection_source = config.database_class.return_value
mocked_connection = (
mocked_database_connection_source.return_value
.__enter__.return_value
)
mocked_cursor = mocked_connection.cursor.return_value.__enter__.return_value
# the call to be tested
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.save_processed(a_processed_crash_with_everything_too_long)
eq_(mocked_database_connection_source.call_count, 1)
eq_(mocked_cursor.execute.call_count, 5)
# check correct fragments
first_call = mocked_cursor.execute.call_args_list[0]
eq_(
first_call[0][1],
a_processed_report_with_everything_truncated * 2
)
def test_basic_key_error_on_save_processed(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres
}],
argv_source=[]
)
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
database = crashstorage.database.return_value = mock.MagicMock()
assert isinstance(database, mock.Mock)
broken_processed_crash = {
"product": "Peter",
"version": "1.0B3",
"ooid": "abc123",
"submitted_timestamp": time.time(),
"unknown_field": 'whatever'
}
with pytest.raises(KeyError):
crashstorage.save_processed(broken_processed_crash)
def test_basic_postgres_save_processed_success(self):
config = DotDict()
config.database_class = mock.MagicMock()
config.transaction_executor_class = TransactionExecutorWithInfiniteBackoff
config.redactor_class = mock.Mock()
config.backoff_delays = [1]
config.wait_log_interval = 10
config.logger = mock.Mock()
mocked_database_connection_source = config.database_class.return_value
mocked_connection = (
mocked_database_connection_source.return_value
.__enter__.return_value
)
mocked_cursor = mocked_connection.cursor.return_value.__enter__.return_value
# the call to be tested
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.save_processed(a_processed_crash)
eq_(mocked_database_connection_source.call_count, 1)
eq_(mocked_cursor.execute.call_count, 5)
# check correct fragments
sql_fragments = [
"UPDATE reports_20120402",
'select id from plugins',
'delete from plugins_reports_20120402',
'insert into plugins_reports_20120402',
'UPDATE processed_crashes_20120402'
]
for a_call, a_fragment in zip(mocked_cursor.execute.call_args_list, sql_fragments):
ok_(a_fragment in a_call[0][0])
def test_basic_postgres_save_processed_success(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres
}]
)
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
database = crashstorage.database.return_value = mock.MagicMock()
self.assertTrue(isinstance(database, mock.Mock))
crashstorage.save_processed(a_processed_crash)
fetch_all_returns = [((666,),), ((23,),), ]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.return_value.fetchall.side_effect=fetch_all_func
crashstorage.save_processed(a_processed_crash)
self.assertEqual(m.cursor.call_count, 6)
self.assertEqual(m.cursor().fetchall.call_count, 2)
self.assertEqual(m.cursor().execute.call_count, 6)
expected_execute_args = (
(('savepoint MainThread', None),),
(('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id',
[None, '0x1c', '...', '20120309050057', '2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884', 'None | 0', 'arm', '2012-04-08 10:56:41.558922', None, None, '*****@*****.**', 'high', '[blank]', None, 22385, None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ', 'SignatureTool: signature truncated due to length', 'plugin', 'FennecAndroid', 'SIGSEGV', 'default', 'libxul.so@0x117441c', '2012-04-08 10:56:50.440752', True, [], False, 170, None, None, 'http://embarasing.p**n.com', '936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1']),),
(('release savepoint MainThread', None),),
(('select id from plugins where filename = %s and name = %s',
('dwight.txt', 'wilma')),),
(('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)',
(666, 23, '2012-04-08 10:56:41.558922', '69')),),
(('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)',
(666, '2012-04-08 10:56:41.558922', 0, '{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')),),
(('release savepoint MainThread', None),),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
self.assertEqual(expected, actual)
def test_save_processed(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
# data doesn't contain an 'ooid' key
crashstorage.save_processed(a_processed_crash)
cursor = self.conn.cursor()
cursor.execute('select uuid from %s' % self.reports_table_name)
report, = cursor.fetchall()
uuid, = report
eq_(uuid, a_processed_crash['uuid'])
def test_save_processed(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
# data doesn't contain an 'ooid' key
crashstorage.save_processed(a_processed_crash)
cursor = self.conn.cursor()
cursor.execute('select uuid from %s' % self.reports_table_name)
report, = cursor.fetchall()
uuid, = report
eq_(uuid, a_processed_crash['uuid'])
def test_postgres_save_processed_success_2(self):
config = DotDict()
config.database_class = mock.MagicMock()
config.transaction_executor_class = TransactionExecutorWithInfiniteBackoff
config.redactor_class = mock.Mock()
config.backoff_delays = [1]
config.wait_log_interval = 10
config.logger = mock.Mock()
mocked_database_connection_source = config.database_class.return_value
mocked_connection = (mocked_database_connection_source.return_value.
__enter__.return_value)
mocked_cursor = mocked_connection.cursor.return_value.__enter__.return_value
fetch_all_returns = [
((666, ), ),
None,
((23, ), ),
]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
mocked_cursor.fetchall = fetch_all_func
# the call to be tested
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.save_processed(a_processed_crash)
assert mocked_database_connection_source.call_count == 1
assert mocked_cursor.execute.call_count == 3
# check correct fragments
sql_fragments = [
"UPDATE reports_20120402",
'select id from plugins',
'insert into plugins',
]
for a_call, a_fragment in zip(mocked_cursor.execute.call_args_list,
sql_fragments):
assert a_fragment in a_call[0][0]
def test_basic_postgres_save_processed_operational_error(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option(
'logger',
default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}],
argv_source=[]
)
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.database.operational_exceptions = (OperationalError,)
database = crashstorage.database.return_value = mock.MagicMock()
assert isinstance(database, mock.Mock)
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.side_effect = OperationalError('bad')
with pytest.raises(OperationalError):
crashstorage.save_processed(a_processed_crash)
assert m.cursor.call_count == 3
def test_save_processed(self):
mock_logging = mock.Mock()
required_config = PostgreSQLCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
}, DSN])
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
# data doesn't contain an 'ooid' key
crashstorage.save_processed(a_processed_crash)
cursor = self.conn.cursor()
cursor.execute('select uuid from %s' % self.reports_table_name)
report, = cursor.fetchall()
uuid, = report
self.assertEqual(uuid, a_processed_crash['uuid'])
def test_save_processed(self):
mock_logging = mock.Mock()
required_config = PostgreSQLCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
}, DSN]
)
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
# data doesn't contain an 'ooid' key
crashstorage.save_processed(a_processed_crash)
cursor = self.conn.cursor()
cursor.execute('select uuid from %s' % self.reports_table_name)
report, = cursor.fetchall()
uuid, = report
self.assertEqual(uuid, a_processed_crash['uuid'])
def test_basic_postgres_save_processed_succeed_after_failures(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}],
argv_source=[]
)
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.database.operational_exceptions = (OperationalError,)
database = crashstorage.database.return_value = mock.MagicMock()
ok_(isinstance(database, mock.Mock))
fetch_all_returns = [((666,),), None, ((23,),), ]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
fetch_mock = mock.Mock()
fetch_mock.fetchall.side_effect = fetch_all_func
connection_trouble = [OperationalError('bad'),
OperationalError('worse'),
]
def broken_connection(*args):
try:
result = connection_trouble.pop(0)
raise result
except IndexError:
return fetch_mock
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.side_effect = broken_connection
crashstorage.save_processed(a_processed_crash)
eq_(m.cursor.call_count, 10)
eq_(m.cursor().fetchall.call_count, 3)
eq_(m.cursor().execute.call_count, 8)
expected_execute_args = (
(('WITH update_report AS (UPDATE reports_20120402 SET addons_checked=%s, address=%s, app_notes=%s, build=%s, client_crash_date=%s, completed_datetime=%s, cpu_info=%s, cpu_name=%s, date_processed=%s, distributor=%s, distributor_version=%s, email=%s, exploitability=%s, flash_version=%s, hangid=%s, install_age=%s, last_crash=%s, os_name=%s, os_version=%s, processor_notes=%s, process_type=%s, product=%s, productid=%s, reason=%s, release_channel=%s, signature=%s, started_datetime=%s, success=%s, topmost_filenames=%s, truncated=%s, uptime=%s, user_comments=%s, user_id=%s, url=%s, uuid=%s, version=%s WHERE uuid=%s RETURNING id), insert_report AS (INSERT INTO reports_20120402(addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version)(SELECT%s as addons_checked, %s as address, %s as app_notes, %s as build, %s as client_crash_date, %s as completed_datetime, %s as cpu_info, %s as cpu_name, %s as date_processed, %s as distributor, %s as distributor_version, %s as email, %s as exploitability, %s as flash_version, %s as hangid, %s as install_age, %s as last_crash, %s as os_name, %s as os_version, %s as processor_notes, %s as process_type, %s as product, %s as productid, %s as reason, %s as release_channel, %s as signature, %s as started_datetime, %s as success, %s as topmost_filenames, %s as truncated, %s as uptime, %s as user_comments, %s as user_id, %s as url, %s as uuid, %s as version WHERE NOT EXISTS(SELECT uuid from reports_20120402 WHERE uuid=%s LIMIT 1)) RETURNING id) SELECT * from update_report UNION ALL SELECT * from insert_report',
[None, '0x1c', '...', '20120309050057', '2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884', 'None | 0', 'arm', '2012-04-08 10:56:41.558922', None, None, '*****@*****.**', 'high', '[blank]', None, 22385, None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ', 'SignatureTool: signature truncated due to length', 'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV', 'default', 'libxul.so@0x117441c', '2012-04-08 10:56:50.440752', True, [], False, 170, None, None, 'http://embarrassing.p**n.com', '936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1', '936ce666-ff3b-4c7a-9674-367fe2120408',
None, '0x1c', '...', '20120309050057', '2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884', 'None | 0', 'arm', '2012-04-08 10:56:41.558922', None, None, '*****@*****.**', 'high', '[blank]', None, 22385, None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ', 'SignatureTool: signature truncated due to length', 'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV', 'default', 'libxul.so@0x117441c', '2012-04-08 10:56:50.440752', True, [], False, 170, None, None, 'http://embarrassing.p**n.com', '936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1', '936ce666-ff3b-4c7a-9674-367fe2120408']),),
(('select id from plugins where filename = %s and name = %s',
('dwight.txt', 'wilma')),),
(('insert into plugins (filename, name) values (%s, %s) returning id',
('dwight.txt', 'wilma')),),
(('delete from plugins_reports_20120402 where report_id = %s',
(666, )),),
(('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)',
(666, 23, '2012-04-08 10:56:41.558922', '69')),),
(('delete from extensions_20120402 where report_id = %s',
(666, )),),
(('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)',
(666, '2012-04-08 10:56:41.558922', 0, '{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')),),
(("""WITH update_processed_crash AS ( UPDATE processed_crashes_20120402 SET processed_crash = %(processed_json)s, date_processed = %(date_processed)s WHERE uuid = %(uuid)s RETURNING 1), insert_processed_crash AS ( INSERT INTO processed_crashes_20120402 (uuid, processed_crash, date_processed) ( SELECT %(uuid)s as uuid, %(processed_json)s as processed_crash, %(date_processed)s as date_processed WHERE NOT EXISTS ( SELECT uuid from processed_crashes_20120402 WHERE uuid = %(uuid)s LIMIT 1)) RETURNING 2) SELECT * from update_processed_crash UNION ALL SELECT * from insert_processed_crash """,
{'uuid': '936ce666-ff3b-4c7a-9674-367fe2120408', 'processed_json': '{"startedDateTime": "2012-04-08 10:56:50.440752", "crashedThread": 8, "cpu_info": "None | 0", "PluginName": "wilma", "install_age": 22385, "topmost_filenames": [], "user_comments": null, "user_id": null, "uuid": "936ce666-ff3b-4c7a-9674-367fe2120408", "flash_version": "[blank]", "os_version": "0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ", "PluginVersion": "69", "addons_checked": null, "completeddatetime": "2012-04-08 10:56:50.902884", "productid": "FA-888888", "success": true, "exploitability": "high", "client_crash_date": "2012-04-08 10:52:42.0", "PluginFilename": "dwight.txt", "dump": "...", "truncated": false, "product": "FennecAndroid", "distributor": null, "processor_notes": "SignatureTool: signature truncated due to length", "uptime": 170, "release_channel": "default", "distributor_version": null, "process_type": "plugin", "id": 361399767, "hangid": null, "version": "13.0a1", "build": "20120309050057", "ReleaseChannel": "default", "email": "*****@*****.**", "app_notes": "...", "os_name": "Linux", "last_crash": null, "date_processed": "2012-04-08 10:56:41.558922", "cpu_name": "arm", "reason": "SIGSEGV", "address": "0x1c", "url": "http://embarrassing.p**n.com", "signature": "libxul.so@0x117441c", "addons": [["{1a5dabbd-0e74-41da-b532-a364bb552cab}", "1.0.4.1"]]}', 'date_processed': '2012-04-08 10:56:41.558922'}),),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
expected_sql, expected_params = expected[0]
expected_sql = remove_whitespace(expected_sql)
actual_sql, actual_params = actual[0]
actual_sql = remove_whitespace(actual_sql)
eq_(expected_sql, actual_sql)
eq_(expected_params, actual_params)
def test_basic_postgres_save_processed_succeed_after_failures(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}])
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.database.operational_exceptions = (OperationalError, )
database = crashstorage.database.return_value = mock.MagicMock()
self.assertTrue(isinstance(database, mock.Mock))
fetch_all_returns = [
((666, ), ),
None,
((23, ), ),
]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
fetch_mock = mock.Mock()
fetch_mock.fetchall.side_effect = fetch_all_func
connection_trouble = [
OperationalError('bad'),
OperationalError('worse'),
]
def broken_connection(*args):
try:
result = connection_trouble.pop(0)
raise result
except IndexError:
return fetch_mock
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.side_effect = broken_connection
crashstorage.save_processed(a_processed_crash)
self.assertEqual(m.cursor.call_count, 9)
self.assertEqual(m.cursor().fetchall.call_count, 3)
self.assertEqual(m.cursor().execute.call_count, 7)
expected_execute_args = (
(('savepoint MainThread', None), ),
(('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id',
[
None, '0x1c', '...', '20120309050057',
'2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884',
'None | 0', 'arm', '2012-04-08 10:56:41.558922', None,
None, '*****@*****.**', 'high', '[blank]', None, 22385,
None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ',
'SignatureTool: signature truncated due to length',
'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV',
'default', 'libxul.so@0x117441c',
'2012-04-08 10:56:50.440752', True, [], False, 170, None,
None, 'http://embarrassing.p**n.com',
'936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1'
]), ),
(('release savepoint MainThread', None), ),
(('select id from plugins where filename = %s and name = %s',
('dwight.txt', 'wilma')), ),
(('insert into plugins (filename, name) values (%s, %s) returning id',
('dwight.txt', 'wilma')), ),
(('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)',
(666, 23, '2012-04-08 10:56:41.558922', '69')), ),
(('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)',
(666, '2012-04-08 10:56:41.558922', 0,
'{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')), ),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
self.assertEqual(expected, actual)
def test_basic_postgres_save_processed_succeed_after_failures(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.required_config
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="testapp",
app_version="1.0",
app_description="app description",
values_source_list=[
{
"logger": mock_logging,
"database_class": mock_postgres,
"transaction_executor_class": TransactionExecutorWithLimitedBackoff,
"backoff_delays": [0, 0, 0],
}
],
)
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.database.operational_exceptions = (OperationalError,)
database = crashstorage.database.return_value = mock.MagicMock()
self.assertTrue(isinstance(database, mock.Mock))
fetch_all_returns = [((666,),), None, ((23,),)]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
fetch_mock = mock.Mock()
fetch_mock.fetchall.side_effect = fetch_all_func
connection_trouble = [OperationalError("bad"), OperationalError("worse")]
def broken_connection(*args):
try:
result = connection_trouble.pop(0)
raise result
except IndexError:
return fetch_mock
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.side_effect = broken_connection
crashstorage.save_processed(a_processed_crash)
self.assertEqual(m.cursor.call_count, 9)
self.assertEqual(m.cursor().fetchall.call_count, 3)
self.assertEqual(m.cursor().execute.call_count, 7)
expected_execute_args = (
(("savepoint MainThread", None),),
(
(
"insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id",
[
None,
"0x1c",
"...",
"20120309050057",
"2012-04-08 10:52:42.0",
"2012-04-08 10:56:50.902884",
"None | 0",
"arm",
"2012-04-08 10:56:41.558922",
None,
None,
"*****@*****.**",
"high",
"[blank]",
None,
22385,
None,
"Linux",
"0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ",
"SignatureTool: signature truncated due to length",
"plugin",
"FennecAndroid",
"FA-888888",
"SIGSEGV",
"default",
"libxul.so@0x117441c",
"2012-04-08 10:56:50.440752",
True,
[],
False,
170,
None,
None,
"http://embarrassing.p**n.com",
"936ce666-ff3b-4c7a-9674-367fe2120408",
"13.0a1",
],
),
),
(("release savepoint MainThread", None),),
(("select id from plugins where filename = %s and name = %s", ("dwight.txt", "wilma")),),
(("insert into plugins (filename, name) values (%s, %s) returning id", ("dwight.txt", "wilma")),),
(
(
"insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)",
(666, 23, "2012-04-08 10:56:41.558922", "69"),
),
),
(
(
"insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)",
(666, "2012-04-08 10:56:41.558922", 0, "{1a5dabbd-0e74-41da-b532-a364bb552cab}", "1.0.4.1"),
),
),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args, actual_execute_args):
self.assertEqual(expected, actual)
def test_basic_postgres_save_processed_succeed_after_failures(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}],
argv_source=[])
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.database.operational_exceptions = (OperationalError, )
database = crashstorage.database.return_value = mock.MagicMock()
ok_(isinstance(database, mock.Mock))
fetch_all_returns = [
((666, ), ),
None,
((23, ), ),
]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
fetch_mock = mock.Mock()
fetch_mock.fetchall.side_effect = fetch_all_func
connection_trouble = [
OperationalError('bad'),
OperationalError('worse'),
]
def broken_connection(*args):
try:
result = connection_trouble.pop(0)
raise result
except IndexError:
return fetch_mock
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.side_effect = broken_connection
crashstorage.save_processed(a_processed_crash)
eq_(m.cursor.call_count, 8)
eq_(m.cursor().fetchall.call_count, 3)
eq_(m.cursor().execute.call_count, 6)
expected_execute_args = (
(('WITH update_report AS (UPDATE reports_20120402 SET addons_checked=%s, address=%s, app_notes=%s, build=%s, client_crash_date=%s, completed_datetime=%s, cpu_info=%s, cpu_name=%s, date_processed=%s, distributor=%s, distributor_version=%s, email=%s, exploitability=%s, flash_version=%s, hangid=%s, install_age=%s, last_crash=%s, os_name=%s, os_version=%s, processor_notes=%s, process_type=%s, product=%s, productid=%s, reason=%s, release_channel=%s, signature=%s, started_datetime=%s, success=%s, topmost_filenames=%s, truncated=%s, uptime=%s, user_comments=%s, user_id=%s, url=%s, uuid=%s, version=%s WHERE uuid=%s RETURNING id), insert_report AS (INSERT INTO reports_20120402(addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version)(SELECT%s as addons_checked, %s as address, %s as app_notes, %s as build, %s as client_crash_date, %s as completed_datetime, %s as cpu_info, %s as cpu_name, %s as date_processed, %s as distributor, %s as distributor_version, %s as email, %s as exploitability, %s as flash_version, %s as hangid, %s as install_age, %s as last_crash, %s as os_name, %s as os_version, %s as processor_notes, %s as process_type, %s as product, %s as productid, %s as reason, %s as release_channel, %s as signature, %s as started_datetime, %s as success, %s as topmost_filenames, %s as truncated, %s as uptime, %s as user_comments, %s as user_id, %s as url, %s as uuid, %s as version WHERE NOT EXISTS(SELECT uuid from reports_20120402 WHERE uuid=%s LIMIT 1)) RETURNING id) SELECT * from update_report UNION ALL SELECT * from insert_report',
[
None, '0x1c', '...', '20120309050057',
'2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884',
'None | 0', 'arm', '2012-04-08 10:56:41.558922', None,
None, '*****@*****.**', 'high', '[blank]', None, 22385,
None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ',
'SignatureTool: signature truncated due to length',
'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV',
'default', 'libxul.so@0x117441c',
'2012-04-08 10:56:50.440752', True, [], False, 170, None,
None, 'http://embarrassing.p**n.com',
'936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1',
'936ce666-ff3b-4c7a-9674-367fe2120408', None, '0x1c',
'...', '20120309050057', '2012-04-08 10:52:42.0',
'2012-04-08 10:56:50.902884', 'None | 0', 'arm',
'2012-04-08 10:56:41.558922', None, None,
'*****@*****.**', 'high', '[blank]', None, 22385, None,
'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ',
'SignatureTool: signature truncated due to length',
'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV',
'default', 'libxul.so@0x117441c',
'2012-04-08 10:56:50.440752', True, [], False, 170, None,
None, 'http://embarrassing.p**n.com',
'936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1',
'936ce666-ff3b-4c7a-9674-367fe2120408'
]), ),
(('select id from plugins where filename = %s and name = %s',
('dwight.txt', 'wilma')), ),
(('insert into plugins (filename, name) values (%s, %s) returning id',
('dwight.txt', 'wilma')), ),
(('delete from plugins_reports_20120402 where report_id = %s',
(666, )), ),
(('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)',
(666, 23, '2012-04-08 10:56:41.558922', '69')), ),
(("""WITH update_processed_crash AS ( UPDATE processed_crashes_20120402 SET processed_crash = %(processed_json)s, date_processed = %(date_processed)s WHERE uuid = %(uuid)s RETURNING 1), insert_processed_crash AS ( INSERT INTO processed_crashes_20120402 (uuid, processed_crash, date_processed) ( SELECT %(uuid)s as uuid, %(processed_json)s as processed_crash, %(date_processed)s as date_processed WHERE NOT EXISTS ( SELECT uuid from processed_crashes_20120402 WHERE uuid = %(uuid)s LIMIT 1)) RETURNING 2) SELECT * from update_processed_crash UNION ALL SELECT * from insert_processed_crash """,
{
'uuid': '936ce666-ff3b-4c7a-9674-367fe2120408',
'processed_json':
'{"startedDateTime": "2012-04-08 10:56:50.440752", "crashedThread": 8, "cpu_info": "None | 0", "PluginName": "wilma", "install_age": 22385, "topmost_filenames": [], "user_comments": null, "user_id": null, "uuid": "936ce666-ff3b-4c7a-9674-367fe2120408", "flash_version": "[blank]", "os_version": "0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ", "PluginVersion": "69", "addons_checked": null, "completeddatetime": "2012-04-08 10:56:50.902884", "productid": "FA-888888", "success": true, "exploitability": "high", "client_crash_date": "2012-04-08 10:52:42.0", "PluginFilename": "dwight.txt", "dump": "...", "truncated": false, "product": "FennecAndroid", "distributor": null, "processor_notes": "SignatureTool: signature truncated due to length", "uptime": 170, "release_channel": "default", "distributor_version": null, "process_type": "plugin", "id": 361399767, "hangid": null, "version": "13.0a1", "build": "20120309050057", "ReleaseChannel": "default", "email": "*****@*****.**", "app_notes": "...", "os_name": "Linux", "last_crash": null, "date_processed": "2012-04-08 10:56:41.558922", "cpu_name": "arm", "reason": "SIGSEGV", "address": "0x1c", "url": "http://embarrassing.p**n.com", "signature": "libxul.so@0x117441c", "addons": [["{1a5dabbd-0e74-41da-b532-a364bb552cab}", "1.0.4.1"]]}',
'date_processed': '2012-04-08 10:56:41.558922'
}), ),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
expected_sql, expected_params = expected[0]
expected_sql = remove_whitespace(expected_sql)
actual_sql, actual_params = actual[0]
actual_sql = remove_whitespace(actual_sql)
eq_(expected_sql, actual_sql)
eq_(expected_params, actual_params)
def test_basic_postgres_save_processed_success2(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres
}],
argv_source=[]
)
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
database = crashstorage.database.return_value = mock.MagicMock()
self.assertTrue(isinstance(database, mock.Mock))
fetch_all_returns = [((666,),), None, ((23,),), ]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.return_value.fetchall.side_effect=fetch_all_func
crashstorage.save_processed(a_processed_crash)
self.assertEqual(m.cursor.call_count, 10)
self.assertEqual(m.cursor().fetchall.call_count, 3)
self.assertEqual(m.cursor().execute.call_count, 10)
expected_execute_args = (
(('savepoint MainThread', None),),
(('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id',
[None, '0x1c', '...', '20120309050057', '2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884', 'None | 0', 'arm', '2012-04-08 10:56:41.558922', None, None, '*****@*****.**', 'high', '[blank]', None, 22385, None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ', 'SignatureTool: signature truncated due to length', 'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV', 'default', 'libxul.so@0x117441c', '2012-04-08 10:56:50.440752', True, [], False, 170, None, None, 'http://embarrassing.p**n.com', '936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1']),),
(('release savepoint MainThread', None),),
(('select id from plugins where filename = %s and name = %s',
('dwight.txt', 'wilma')),),
(('insert into plugins (filename, name) values (%s, %s) returning id',
('dwight.txt', 'wilma')),),
(('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)',
(666, 23, '2012-04-08 10:56:41.558922', '69')),),
(('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)',
(666, '2012-04-08 10:56:41.558922', 0, '{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')),),
(('savepoint MainThread', None),),
(('insert into processed_crashes_20120402 (uuid, processed_crash, date_processed) values (%s, %s, %s)',
('936ce666-ff3b-4c7a-9674-367fe2120408', '{"startedDateTime": "2012-04-08 10:56:50.440752", "crashedThread": 8, "cpu_info": "None | 0", "PluginName": "wilma", "install_age": 22385, "topmost_filenames": [], "user_comments": null, "user_id": null, "uuid": "936ce666-ff3b-4c7a-9674-367fe2120408", "flash_version": "[blank]", "os_version": "0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ", "PluginVersion": "69", "addons_checked": null, "completeddatetime": "2012-04-08 10:56:50.902884", "productid": "FA-888888", "success": true, "exploitability": "high", "client_crash_date": "2012-04-08 10:52:42.0", "PluginFilename": "dwight.txt", "dump": "...", "truncated": false, "product": "FennecAndroid", "distributor": null, "processor_notes": "SignatureTool: signature truncated due to length", "uptime": 170, "release_channel": "default", "distributor_version": null, "process_type": "plugin", "id": 361399767, "hangid": null, "version": "13.0a1", "build": "20120309050057", "ReleaseChannel": "default", "email": "*****@*****.**", "app_notes": "...", "os_name": "Linux", "last_crash": null, "date_processed": "2012-04-08 10:56:41.558922", "cpu_name": "arm", "reason": "SIGSEGV", "address": "0x1c", "url": "http://embarrassing.p**n.com", "signature": "libxul.so@0x117441c", "addons": [["{1a5dabbd-0e74-41da-b532-a364bb552cab}", "1.0.4.1"]]}', '2012-04-08 10:56:41.558922')
),),
(('release savepoint MainThread', None),),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
expected_sql, expected_params = expected[0]
expected_sql = expected_sql.replace('\n', '')
expected_sql = expected_sql.replace(' ', '')
actual_sql, actual_params = actual[0]
actual_sql = actual_sql.replace('\n', '')
actual_sql = actual_sql.replace(' ', '')
self.assertEqual(expected_sql, actual_sql)
self.assertEqual(expected_params, actual_params)
