def setUp(self):
super(TestCleanupRadix, self).setUp()
self.temp_fs_root = tempfile.mkdtemp()
with self._setup_radix_storage().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def setUp(self):
super(TestCleanupRadix, self).setUp()
self.temp_fs_root = tempfile.mkdtemp()
with self._setup_radix_storage().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def setUp(self):
super(TestCleanupRadix, self).setUp()
self.psycopg2_patcher = mock.patch("psycopg2.connect")
self.mocked_connection = mock.Mock()
self.psycopg2 = self.psycopg2_patcher.start()
with self._setup_radix_storage().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def _setup_radix_storage(self):
mock_logging = mock.Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="testapp",
app_version="1.0",
app_description="app description",
values_source_list=[{"logger": mock_logging, "minute_slice_interval": 1}],
)
return config_manager
def _common_config_setup(self):
mock_logging = Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1
}])
return config_manager
def _common_config_setup(self):
mock_logging = Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1
}]
)
return config_manager
def _setup_radix_storage(self):
mock_logging = mock.Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1,
'fs_root': self.temp_fs_root
}],
argv_source=[]
)
return config_manager
def setUp(self):
super(TestFSDatedRadixTreeStorage, self).setUp()
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
class TestFSDatedRadixTreeStorage(TestCase):
CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025"
CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026"
def setUp(self):
super(TestFSDatedRadixTreeStorage, self).setUp()
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
super(TestFSDatedRadixTreeStorage, self).tearDown()
shutil.rmtree(self.fsrts.config.fs_root)
def _common_config_setup(self):
mock_logging = Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1
}],
argv_source=[]
)
return config_manager
def _make_test_crash(self):
self.fsrts.save_raw_crash(
{ # raw crash
"test": "TEST"
},
MemoryDumpsMapping({ # dumps
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}),
self.CRASH_ID_1
)
def test_save_raw_crash(self):
try:
self._make_test_crash()
ok_(os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
ok_(os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
ok_(
os.path.exists(
"%s/%s.json" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(
os.path.exists(
"%s/%s.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(
os.path.exists(
"%s/%s.foo.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_save_raw_crash_no_dumps(self):
try:
self.fsrts.save_raw_crash(
{ # raw crash
"test": "TEST"
},
None, # dumps
self.CRASH_ID_1
)
ok_(os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
ok_(os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
ok_(
os.path.exists(
"%s/%s.json" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.foo.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_save_raw_and_processed_crash_no_dumps(self):
try:
self.fsrts.save_raw_and_processed(
{ # raw crash
"test": "TEST"
},
None, # dumps
{ # processed_crash
'processed': 'crash',
'uuid': self.CRASH_ID_1,
},
self.CRASH_ID_1
)
ok_(os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
ok_(os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
ok_(
os.path.exists(
"%s/%s.json" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.foo.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(
os.path.exists(
"%s/%s.jsonz" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_get_raw_crash(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'],
"TEST")
assert_raises(CrashIDNotFound, self.fsrts.get_raw_crash,
self.CRASH_ID_2)
def test_get_raw_dump(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'),
"bar")
eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1,
self.fsrts.config.dump_field),
"baz")
assert_raises(CrashIDNotFound, self.fsrts.get_raw_dump,
self.CRASH_ID_2, "foo")
assert_raises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1,
"foor")
def test_get_raw_dumps(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_dumps(self.CRASH_ID_1), {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
})
assert_raises(CrashIDNotFound, self.fsrts.get_raw_dumps,
self.CRASH_ID_2)
def test_remove(self):
self._make_test_crash()
self.fsrts.remove(self.CRASH_ID_1)
parent = os.path.realpath(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1)))
p = os.path.join(parent, self.CRASH_ID_1)
ok_(not os.path.exists(p))
p = os.path.dirname(p)
ok_(not os.path.exists(p))
p = os.path.dirname(p)
ok_(not os.path.exists(p))
assert_raises(CrashIDNotFound, self.fsrts.remove,
self.CRASH_ID_2)
def test_new_crashes(self):
self.fsrts._current_slot = lambda: ['00', '00_00']
self._make_test_crash()
self.fsrts._current_slot = lambda: ['00', '00_01']
eq_(list(self.fsrts.new_crashes()), [self.CRASH_ID_1])
eq_(list(self.fsrts.new_crashes()), [])
self.fsrts.remove(self.CRASH_ID_1)
del self.fsrts._current_slot
def setUp(self):
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
class TestFSDatedRadixTreeStorage(unittest.TestCase):
CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025"
CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026"
def setUp(self):
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
shutil.rmtree(self.fsrts.config.fs_root)
def _common_config_setup(self):
mock_logging = Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1
}]
)
return config_manager
def _make_test_crash(self):
self.fsrts.save_raw_crash({
"test": "TEST"
}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, self.CRASH_ID_1)
def test_save_raw_crash(self):
self._make_test_crash()
self.assertTrue(os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
self.assertTrue(os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
def test_get_raw_crash(self):
self._make_test_crash()
self.assertEqual(self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'],
"TEST")
self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_crash,
self.CRASH_ID_2)
def test_get_raw_dump(self):
self._make_test_crash()
self.assertEqual(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'),
"bar")
self.assertEqual(self.fsrts.get_raw_dump(self.CRASH_ID_1,
self.fsrts.config.dump_field),
"baz")
self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dump,
self.CRASH_ID_2, "foo")
self.assertRaises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1,
"foor")
def test_get_raw_dumps(self):
self._make_test_crash()
self.assertEqual(self.fsrts.get_raw_dumps(self.CRASH_ID_1), {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
})
self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dumps,
self.CRASH_ID_2)
def test_remove(self):
self._make_test_crash()
self.fsrts.remove(self.CRASH_ID_1)
parent = os.path.realpath(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1)))
p = os.path.join(parent, self.CRASH_ID_1)
self.assertTrue(not os.path.exists(p))
p = os.path.dirname(p)
self.assertTrue(not os.path.exists(p))
p = os.path.dirname(p)
self.assertTrue(not os.path.exists(p))
self.assertRaises(CrashIDNotFound, self.fsrts.remove,
self.CRASH_ID_2)
def test_new_crashes(self):
self.fsrts._current_slot = lambda: ['00', '00_00']
self._make_test_crash()
self.fsrts._current_slot = lambda: ['00', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID_1])
self.assertEqual(list(self.fsrts.new_crashes()), [])
self.fsrts.remove(self.CRASH_ID_1)
del self.fsrts._current_slot
def setUp(self):
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
class TestFSDatedRadixTreeStorage(TestCase):
CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025"
CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026"
def setUp(self):
super(TestFSDatedRadixTreeStorage, self).setUp()
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
super(TestFSDatedRadixTreeStorage, self).tearDown()
shutil.rmtree(self.fsrts.config.fs_root)
def _common_config_setup(self):
mock_logging = Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="testapp",
app_version="1.0",
app_description="app description",
values_source_list=[{"logger": mock_logging, "minute_slice_interval": 1}],
argv_source=[],
)
return config_manager
def _make_test_crash(self):
self.fsrts.save_raw_crash(
{"test": "TEST"}, # raw crash
MemoryDumpsMapping({"foo": "bar", self.fsrts.config.dump_field: "baz"}), # dumps
self.CRASH_ID_1,
)
def test_save_raw_crash(self):
try:
self._make_test_crash()
ok_(
os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
)
)
)
ok_(
os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1,
)
)
)
ok_(
os.path.exists(
"%s/%s.json" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
ok_(
os.path.exists(
"%s/%s.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
ok_(
os.path.exists(
"%s/%s.foo.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_save_raw_crash_no_dumps(self):
try:
self.fsrts.save_raw_crash({"test": "TEST"}, None, self.CRASH_ID_1) # raw crash # dumps
ok_(
os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
)
)
)
ok_(
os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1,
)
)
)
ok_(
os.path.exists(
"%s/%s.json" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
ok_(
not os.path.exists(
"%s/%s.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
ok_(
not os.path.exists(
"%s/%s.foo.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_save_raw_and_processed_crash_no_dumps(self):
try:
self.fsrts.save_raw_and_processed(
{"test": "TEST"}, # raw crash
None, # dumps
{"processed": "crash", "uuid": self.CRASH_ID_1}, # processed_crash
self.CRASH_ID_1,
)
ok_(
os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
)
)
)
ok_(
os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1,
)
)
)
ok_(
os.path.exists(
"%s/%s.json" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
ok_(
not os.path.exists(
"%s/%s.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
ok_(
not os.path.exists(
"%s/%s.foo.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
ok_(
os.path.exists(
"%s/%s.jsonz" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_get_raw_crash(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_crash(self.CRASH_ID_1)["test"], "TEST")
assert_raises(CrashIDNotFound, self.fsrts.get_raw_crash, self.CRASH_ID_2)
def test_get_raw_dump(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, "foo"), "bar")
eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, self.fsrts.config.dump_field), "baz")
assert_raises(CrashIDNotFound, self.fsrts.get_raw_dump, self.CRASH_ID_2, "foo")
assert_raises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1, "foor")
def test_get_raw_dumps(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_dumps(self.CRASH_ID_1), {"foo": "bar", self.fsrts.config.dump_field: "baz"})
assert_raises(CrashIDNotFound, self.fsrts.get_raw_dumps, self.CRASH_ID_2)
def test_remove(self):
self._make_test_crash()
self.fsrts.remove(self.CRASH_ID_1)
parent = os.path.realpath(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
)
)
p = os.path.join(parent, self.CRASH_ID_1)
ok_(not os.path.exists(p))
p = os.path.dirname(p)
ok_(not os.path.exists(p))
p = os.path.dirname(p)
ok_(not os.path.exists(p))
assert_raises(CrashIDNotFound, self.fsrts.remove, self.CRASH_ID_2)
def test_new_crashes(self):
self.fsrts._current_slot = lambda: ["00", "00_00"]
self._make_test_crash()
self.fsrts._current_slot = lambda: ["00", "00_01"]
eq_(list(self.fsrts.new_crashes()), [self.CRASH_ID_1])
eq_(list(self.fsrts.new_crashes()), [])
self.fsrts.remove(self.CRASH_ID_1)
del self.fsrts._current_slot
class TestCleanupRadix(IntegrationTestCaseBase):
CRASH_ID = "0bba929f-8721-460c-dead-a43c20071025"
def setUp(self):
super(TestCleanupRadix, self).setUp()
self.temp_fs_root = tempfile.mkdtemp()
with self._setup_radix_storage().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
super(TestCleanupRadix, self).tearDown()
shutil.rmtree(self.temp_fs_root)
def _setup_radix_storage(self):
mock_logging = mock.Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1,
'fs_root': self.temp_fs_root
}]
)
return config_manager
def _setup_config_manager(self):
_super = super(TestCleanupRadix, self)._setup_config_manager
return _super(
'socorro.cron.jobs.cleanup_radix.RadixCleanupCronApp|1d',
{
'crontabber.class-RadixCleanupCronApp.dated_storage_classes':
'socorro.external.fs.crashstorage.FSDatedRadixTreeStorage',
'fs_root': self.temp_fs_root,
}
)
def test_cleanup_radix(self):
self.fsrts._current_slot = lambda: ['00', '00_00']
self.fsrts.save_raw_crash({
"test": "TEST"
}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, self.CRASH_ID)
self.fsrts._current_slot = lambda: ['10', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID])
self.assertEqual(list(self.fsrts.new_crashes()), [])
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['cleanup_radix']
assert not information['cleanup_radix']['last_error']
assert information['cleanup_radix']['last_success']
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [])
future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d")
future_id = "0bba929f-8721-460c-dead-a43c%s" % future
self.fsrts._current_slot = lambda: ['00', '00_00']
self.fsrts.save_raw_crash({
"test": "TEST"
}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, future_id)
self.fsrts._current_slot = lambda: ['10', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [future_id])
self.assertEqual(list(self.fsrts.new_crashes()), [])
tab.run_all()
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])
class TestCleanupRadix(TestCaseBase):
CRASH_ID = "0bba929f-8721-460c-dead-a43c20071025"
def setUp(self):
super(TestCleanupRadix, self).setUp()
self.psycopg2_patcher = mock.patch("psycopg2.connect")
self.mocked_connection = mock.Mock()
self.psycopg2 = self.psycopg2_patcher.start()
with self._setup_radix_storage().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
super(TestCleanupRadix, self).tearDown()
self.psycopg2_patcher.stop()
shutil.rmtree(self.fsrts.config.fs_root)
def _setup_radix_storage(self):
mock_logging = mock.Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="testapp",
app_version="1.0",
app_description="app description",
values_source_list=[{"logger": mock_logging, "minute_slice_interval": 1}],
)
return config_manager
def _setup_config_manager(self):
_super = super(TestCleanupRadix, self)._setup_config_manager
config_manager, json_file = _super(
"socorro.cron.jobs.cleanup_radix.RadixCleanupCronApp|1d",
{"crontabber.class-RadixCleanupCronApp.storage0": {"crashstorage_class": FSDatedRadixTreeStorage}},
)
return config_manager, json_file
def test_cleanup_radix(self):
self.fsrts._current_slot = lambda: ["00", "00_00"]
self.fsrts.save_raw_crash({"test": "TEST"}, {"foo": "bar", self.fsrts.config.dump_field: "baz"}, self.CRASH_ID)
self.fsrts._current_slot = lambda: ["10", "00_01"]
self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID])
self.assertEqual(list(self.fsrts.new_crashes()), [])
config_manager, json_file = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [])
future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d")
future_id = "0bba929f-8721-460c-dead-a43c%s" % future
self.fsrts._current_slot = lambda: ["00", "00_00"]
self.fsrts.save_raw_crash({"test": "TEST"}, {"foo": "bar", self.fsrts.config.dump_field: "baz"}, future_id)
self.fsrts._current_slot = lambda: ["10", "00_01"]
self.assertEqual(list(self.fsrts.new_crashes()), [future_id])
self.assertEqual(list(self.fsrts.new_crashes()), [])
tab.run_all()
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])
def setUp(self):
super(TestFSDatedRadixTreeStorage, self).setUp()
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
class TestFSDatedRadixTreeStorage(TestCase):
CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025"
CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026"
def setUp(self):
super(TestFSDatedRadixTreeStorage, self).setUp()
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
super(TestFSDatedRadixTreeStorage, self).tearDown()
shutil.rmtree(self.fsrts.config.fs_root)
def _common_config_setup(self):
mock_logging = Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1,
'fs_root': os.environ['resource.fs.fs_root'],
}],
argv_source=[]
)
return config_manager
def _make_test_crash(self):
self.fsrts.save_raw_crash(
{ # raw crash
"test": "TEST"
},
MemoryDumpsMapping({ # dumps
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}),
self.CRASH_ID_1
)
def test_save_raw_crash(self):
try:
self._make_test_crash()
ok_(os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
ok_(os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
ok_(
os.path.exists(
"%s/%s.json" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(
os.path.exists(
"%s/%s.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(
os.path.exists(
"%s/%s.foo.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_save_raw_crash_no_dumps(self):
try:
self.fsrts.save_raw_crash(
{ # raw crash
"test": "TEST"
},
None, # dumps
self.CRASH_ID_1
)
ok_(os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
ok_(os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
ok_(
os.path.exists(
"%s/%s.json" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.foo.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_save_raw_and_processed_crash_no_dumps(self):
try:
self.fsrts.save_raw_and_processed(
{ # raw crash
"test": "TEST"
},
None, # dumps
{ # processed_crash
'processed': 'crash',
'uuid': self.CRASH_ID_1,
},
self.CRASH_ID_1
)
ok_(os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
ok_(os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
ok_(
os.path.exists(
"%s/%s.json" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(not
os.path.exists(
"%s/%s.foo.dump" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
ok_(
os.path.exists(
"%s/%s.jsonz" % (
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.CRASH_ID_1
)
)
)
finally:
self.fsrts.remove(self.CRASH_ID_1)
def test_get_raw_crash(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'],
"TEST")
assert_raises(CrashIDNotFound, self.fsrts.get_raw_crash,
self.CRASH_ID_2)
def test_get_raw_dump(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'),
"bar")
eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1,
self.fsrts.config.dump_field),
"baz")
assert_raises(CrashIDNotFound, self.fsrts.get_raw_dump,
self.CRASH_ID_2, "foo")
assert_raises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1,
"foor")
def test_get_raw_dumps(self):
self._make_test_crash()
eq_(self.fsrts.get_raw_dumps(self.CRASH_ID_1), {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
})
assert_raises(CrashIDNotFound, self.fsrts.get_raw_dumps,
self.CRASH_ID_2)
def test_remove(self):
self._make_test_crash()
self.fsrts.remove(self.CRASH_ID_1)
parent = os.path.realpath(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1)))
p = os.path.join(parent, self.CRASH_ID_1)
ok_(not os.path.exists(p))
p = os.path.dirname(p)
ok_(not os.path.exists(p))
p = os.path.dirname(p)
ok_(not os.path.exists(p))
assert_raises(CrashIDNotFound, self.fsrts.remove,
self.CRASH_ID_2)
def test_new_crashes(self):
self.fsrts._current_slot = lambda: ['00', '00_00']
self._make_test_crash()
self.fsrts._current_slot = lambda: ['00', '00_01']
eq_(list(self.fsrts.new_crashes()), [self.CRASH_ID_1])
eq_(list(self.fsrts.new_crashes()), [])
self.fsrts.remove(self.CRASH_ID_1)
del self.fsrts._current_slot
class TestFSDatedRadixTreeStorage(unittest.TestCase):
CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025"
CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026"
def setUp(self):
with self._common_config_setup().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
shutil.rmtree(self.fsrts.config.fs_root)
def _common_config_setup(self):
mock_logging = Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1
}])
return config_manager
def _make_test_crash(self):
self.fsrts.save_raw_crash({"test": "TEST"}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, self.CRASH_ID_1)
def test_save_raw_crash(self):
self._make_test_crash()
self.assertTrue(
os.path.islink(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1))))
self.assertTrue(
os.path.exists(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1),
self.CRASH_ID_1)))
def test_get_raw_crash(self):
self._make_test_crash()
self.assertEqual(
self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'], "TEST")
self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_crash,
self.CRASH_ID_2)
def test_get_raw_dump(self):
self._make_test_crash()
self.assertEqual(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'),
"bar")
self.assertEqual(
self.fsrts.get_raw_dump(self.CRASH_ID_1,
self.fsrts.config.dump_field), "baz")
self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dump,
self.CRASH_ID_2, "foo")
self.assertRaises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1,
"foor")
def test_get_raw_dumps(self):
self._make_test_crash()
self.assertEqual(self.fsrts.get_raw_dumps(self.CRASH_ID_1), {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
})
self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dumps,
self.CRASH_ID_2)
def test_remove(self):
self._make_test_crash()
self.fsrts.remove(self.CRASH_ID_1)
parent = os.path.realpath(
os.path.join(
self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1),
self.fsrts._get_date_root_name(self.CRASH_ID_1)))
p = os.path.join(parent, self.CRASH_ID_1)
self.assertTrue(not os.path.exists(p))
p = os.path.dirname(p)
self.assertTrue(not os.path.exists(p))
p = os.path.dirname(p)
self.assertTrue(not os.path.exists(p))
self.assertRaises(CrashIDNotFound, self.fsrts.remove, self.CRASH_ID_2)
def test_new_crashes(self):
self.fsrts._current_slot = lambda: ['00', '00_00']
self._make_test_crash()
self.fsrts._current_slot = lambda: ['00', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID_1])
self.assertEqual(list(self.fsrts.new_crashes()), [])
self.fsrts.remove(self.CRASH_ID_1)
del self.fsrts._current_slot
class TestCleanupRadix(IntegrationTestCaseBase):
CRASH_ID = "0bba929f-8721-460c-dead-a43c20071025"
def setUp(self):
super(TestCleanupRadix, self).setUp()
self.temp_fs_root = tempfile.mkdtemp()
with self._setup_radix_storage().context() as config:
self.fsrts = FSDatedRadixTreeStorage(config)
def tearDown(self):
super(TestCleanupRadix, self).tearDown()
shutil.rmtree(self.temp_fs_root)
def _setup_radix_storage(self):
mock_logging = mock.Mock()
required_config = FSDatedRadixTreeStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'minute_slice_interval': 1,
'fs_root': self.temp_fs_root
}])
return config_manager
def _setup_config_manager(self):
_super = super(TestCleanupRadix, self)._setup_config_manager
return _super(
'socorro.cron.jobs.cleanup_radix.RadixCleanupCronApp|1d', {
'crontabber.class-RadixCleanupCronApp.dated_storage_classes':
'socorro.external.fs.crashstorage.FSDatedRadixTreeStorage',
'fs_root': self.temp_fs_root,
})
def test_cleanup_radix(self):
self.fsrts._current_slot = lambda: ['00', '00_00']
self.fsrts.save_raw_crash({"test": "TEST"}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, self.CRASH_ID)
self.fsrts._current_slot = lambda: ['10', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID])
self.assertEqual(list(self.fsrts.new_crashes()), [])
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['cleanup_radix']
assert not information['cleanup_radix']['last_error']
assert information['cleanup_radix']['last_success']
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [])
future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d")
future_id = "0bba929f-8721-460c-dead-a43c%s" % future
self.fsrts._current_slot = lambda: ['00', '00_00']
self.fsrts.save_raw_crash({"test": "TEST"}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, future_id)
self.fsrts._current_slot = lambda: ['10', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [future_id])
self.assertEqual(list(self.fsrts.new_crashes()), [])
tab.run_all()
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])