def setUp(self): super(TestCleanupRadix, self).setUp() self.temp_fs_root = tempfile.mkdtemp() with self._setup_radix_storage().context() as config: self.fsrts = FSDatedRadixTreeStorage(config)
def setUp(self): super(TestCleanupRadix, self).setUp() self.psycopg2_patcher = mock.patch("psycopg2.connect") self.mocked_connection = mock.Mock() self.psycopg2 = self.psycopg2_patcher.start() with self._setup_radix_storage().context() as config: self.fsrts = FSDatedRadixTreeStorage(config)
def _setup_radix_storage(self): mock_logging = mock.Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option("logger", default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[{"logger": mock_logging, "minute_slice_interval": 1}], ) return config_manager
def _common_config_setup(self): mock_logging = Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1 }]) return config_manager
def _common_config_setup(self): mock_logging = Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1 }] ) return config_manager
def _setup_radix_storage(self): mock_logging = mock.Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1, 'fs_root': self.temp_fs_root }], argv_source=[] ) return config_manager
def setUp(self): super(TestFSDatedRadixTreeStorage, self).setUp() with self._common_config_setup().context() as config: self.fsrts = FSDatedRadixTreeStorage(config)
class TestFSDatedRadixTreeStorage(TestCase): CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025" CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026" def setUp(self): super(TestFSDatedRadixTreeStorage, self).setUp() with self._common_config_setup().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): super(TestFSDatedRadixTreeStorage, self).tearDown() shutil.rmtree(self.fsrts.config.fs_root) def _common_config_setup(self): mock_logging = Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1 }], argv_source=[] ) return config_manager def _make_test_crash(self): self.fsrts.save_raw_crash( { # raw crash "test": "TEST" }, MemoryDumpsMapping({ # dumps 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }), self.CRASH_ID_1 ) def test_save_raw_crash(self): try: self._make_test_crash() ok_(os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) ok_(os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) ok_( os.path.exists( "%s/%s.json" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_( os.path.exists( "%s/%s.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_( os.path.exists( "%s/%s.foo.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_save_raw_crash_no_dumps(self): try: self.fsrts.save_raw_crash( { # raw crash "test": "TEST" }, None, # dumps self.CRASH_ID_1 ) ok_(os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) ok_(os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) ok_( os.path.exists( "%s/%s.json" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.foo.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_save_raw_and_processed_crash_no_dumps(self): try: self.fsrts.save_raw_and_processed( { # raw crash "test": "TEST" }, None, # dumps { # processed_crash 'processed': 'crash', 'uuid': self.CRASH_ID_1, }, self.CRASH_ID_1 ) ok_(os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) ok_(os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) ok_( os.path.exists( "%s/%s.json" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.foo.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_( os.path.exists( "%s/%s.jsonz" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_get_raw_crash(self): self._make_test_crash() eq_(self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'], "TEST") assert_raises(CrashIDNotFound, self.fsrts.get_raw_crash, self.CRASH_ID_2) def test_get_raw_dump(self): self._make_test_crash() eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'), "bar") eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, self.fsrts.config.dump_field), "baz") assert_raises(CrashIDNotFound, self.fsrts.get_raw_dump, self.CRASH_ID_2, "foo") assert_raises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1, "foor") def test_get_raw_dumps(self): self._make_test_crash() eq_(self.fsrts.get_raw_dumps(self.CRASH_ID_1), { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }) assert_raises(CrashIDNotFound, self.fsrts.get_raw_dumps, self.CRASH_ID_2) def test_remove(self): self._make_test_crash() self.fsrts.remove(self.CRASH_ID_1) parent = os.path.realpath( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1))) p = os.path.join(parent, self.CRASH_ID_1) ok_(not os.path.exists(p)) p = os.path.dirname(p) ok_(not os.path.exists(p)) p = os.path.dirname(p) ok_(not os.path.exists(p)) assert_raises(CrashIDNotFound, self.fsrts.remove, self.CRASH_ID_2) def test_new_crashes(self): self.fsrts._current_slot = lambda: ['00', '00_00'] self._make_test_crash() self.fsrts._current_slot = lambda: ['00', '00_01'] eq_(list(self.fsrts.new_crashes()), [self.CRASH_ID_1]) eq_(list(self.fsrts.new_crashes()), []) self.fsrts.remove(self.CRASH_ID_1) del self.fsrts._current_slot
def setUp(self): with self._common_config_setup().context() as config: self.fsrts = FSDatedRadixTreeStorage(config)
class TestFSDatedRadixTreeStorage(unittest.TestCase): CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025" CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026" def setUp(self): with self._common_config_setup().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): shutil.rmtree(self.fsrts.config.fs_root) def _common_config_setup(self): mock_logging = Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1 }] ) return config_manager def _make_test_crash(self): self.fsrts.save_raw_crash({ "test": "TEST" }, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, self.CRASH_ID_1) def test_save_raw_crash(self): self._make_test_crash() self.assertTrue(os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) self.assertTrue(os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) def test_get_raw_crash(self): self._make_test_crash() self.assertEqual(self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'], "TEST") self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_crash, self.CRASH_ID_2) def test_get_raw_dump(self): self._make_test_crash() self.assertEqual(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'), "bar") self.assertEqual(self.fsrts.get_raw_dump(self.CRASH_ID_1, self.fsrts.config.dump_field), "baz") self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dump, self.CRASH_ID_2, "foo") self.assertRaises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1, "foor") def test_get_raw_dumps(self): self._make_test_crash() self.assertEqual(self.fsrts.get_raw_dumps(self.CRASH_ID_1), { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }) self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dumps, self.CRASH_ID_2) def test_remove(self): self._make_test_crash() self.fsrts.remove(self.CRASH_ID_1) parent = os.path.realpath( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1))) p = os.path.join(parent, self.CRASH_ID_1) self.assertTrue(not os.path.exists(p)) p = os.path.dirname(p) self.assertTrue(not os.path.exists(p)) p = os.path.dirname(p) self.assertTrue(not os.path.exists(p)) self.assertRaises(CrashIDNotFound, self.fsrts.remove, self.CRASH_ID_2) def test_new_crashes(self): self.fsrts._current_slot = lambda: ['00', '00_00'] self._make_test_crash() self.fsrts._current_slot = lambda: ['00', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID_1]) self.assertEqual(list(self.fsrts.new_crashes()), []) self.fsrts.remove(self.CRASH_ID_1) del self.fsrts._current_slot
class TestFSDatedRadixTreeStorage(TestCase): CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025" CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026" def setUp(self): super(TestFSDatedRadixTreeStorage, self).setUp() with self._common_config_setup().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): super(TestFSDatedRadixTreeStorage, self).tearDown() shutil.rmtree(self.fsrts.config.fs_root) def _common_config_setup(self): mock_logging = Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option("logger", default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[{"logger": mock_logging, "minute_slice_interval": 1}], argv_source=[], ) return config_manager def _make_test_crash(self): self.fsrts.save_raw_crash( {"test": "TEST"}, # raw crash MemoryDumpsMapping({"foo": "bar", self.fsrts.config.dump_field: "baz"}), # dumps self.CRASH_ID_1, ) def test_save_raw_crash(self): try: self._make_test_crash() ok_( os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), ) ) ) ok_( os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1, ) ) ) ok_( os.path.exists( "%s/%s.json" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) ok_( os.path.exists( "%s/%s.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) ok_( os.path.exists( "%s/%s.foo.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_save_raw_crash_no_dumps(self): try: self.fsrts.save_raw_crash({"test": "TEST"}, None, self.CRASH_ID_1) # raw crash # dumps ok_( os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), ) ) ) ok_( os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1, ) ) ) ok_( os.path.exists( "%s/%s.json" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) ok_( not os.path.exists( "%s/%s.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) ok_( not os.path.exists( "%s/%s.foo.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_save_raw_and_processed_crash_no_dumps(self): try: self.fsrts.save_raw_and_processed( {"test": "TEST"}, # raw crash None, # dumps {"processed": "crash", "uuid": self.CRASH_ID_1}, # processed_crash self.CRASH_ID_1, ) ok_( os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), ) ) ) ok_( os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1, ) ) ) ok_( os.path.exists( "%s/%s.json" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) ok_( not os.path.exists( "%s/%s.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) ok_( not os.path.exists( "%s/%s.foo.dump" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) ok_( os.path.exists( "%s/%s.jsonz" % (self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_get_raw_crash(self): self._make_test_crash() eq_(self.fsrts.get_raw_crash(self.CRASH_ID_1)["test"], "TEST") assert_raises(CrashIDNotFound, self.fsrts.get_raw_crash, self.CRASH_ID_2) def test_get_raw_dump(self): self._make_test_crash() eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, "foo"), "bar") eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, self.fsrts.config.dump_field), "baz") assert_raises(CrashIDNotFound, self.fsrts.get_raw_dump, self.CRASH_ID_2, "foo") assert_raises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1, "foor") def test_get_raw_dumps(self): self._make_test_crash() eq_(self.fsrts.get_raw_dumps(self.CRASH_ID_1), {"foo": "bar", self.fsrts.config.dump_field: "baz"}) assert_raises(CrashIDNotFound, self.fsrts.get_raw_dumps, self.CRASH_ID_2) def test_remove(self): self._make_test_crash() self.fsrts.remove(self.CRASH_ID_1) parent = os.path.realpath( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), ) ) p = os.path.join(parent, self.CRASH_ID_1) ok_(not os.path.exists(p)) p = os.path.dirname(p) ok_(not os.path.exists(p)) p = os.path.dirname(p) ok_(not os.path.exists(p)) assert_raises(CrashIDNotFound, self.fsrts.remove, self.CRASH_ID_2) def test_new_crashes(self): self.fsrts._current_slot = lambda: ["00", "00_00"] self._make_test_crash() self.fsrts._current_slot = lambda: ["00", "00_01"] eq_(list(self.fsrts.new_crashes()), [self.CRASH_ID_1]) eq_(list(self.fsrts.new_crashes()), []) self.fsrts.remove(self.CRASH_ID_1) del self.fsrts._current_slot
class TestCleanupRadix(IntegrationTestCaseBase): CRASH_ID = "0bba929f-8721-460c-dead-a43c20071025" def setUp(self): super(TestCleanupRadix, self).setUp() self.temp_fs_root = tempfile.mkdtemp() with self._setup_radix_storage().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): super(TestCleanupRadix, self).tearDown() shutil.rmtree(self.temp_fs_root) def _setup_radix_storage(self): mock_logging = mock.Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1, 'fs_root': self.temp_fs_root }] ) return config_manager def _setup_config_manager(self): _super = super(TestCleanupRadix, self)._setup_config_manager return _super( 'socorro.cron.jobs.cleanup_radix.RadixCleanupCronApp|1d', { 'crontabber.class-RadixCleanupCronApp.dated_storage_classes': 'socorro.external.fs.crashstorage.FSDatedRadixTreeStorage', 'fs_root': self.temp_fs_root, } ) def test_cleanup_radix(self): self.fsrts._current_slot = lambda: ['00', '00_00'] self.fsrts.save_raw_crash({ "test": "TEST" }, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, self.CRASH_ID) self.fsrts._current_slot = lambda: ['10', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID]) self.assertEqual(list(self.fsrts.new_crashes()), []) config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['cleanup_radix'] assert not information['cleanup_radix']['last_error'] assert information['cleanup_radix']['last_success'] self.assertEqual(os.listdir(self.fsrts.config.fs_root), []) future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d") future_id = "0bba929f-8721-460c-dead-a43c%s" % future self.fsrts._current_slot = lambda: ['00', '00_00'] self.fsrts.save_raw_crash({ "test": "TEST" }, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, future_id) self.fsrts._current_slot = lambda: ['10', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [future_id]) self.assertEqual(list(self.fsrts.new_crashes()), []) tab.run_all() self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])
class TestCleanupRadix(TestCaseBase): CRASH_ID = "0bba929f-8721-460c-dead-a43c20071025" def setUp(self): super(TestCleanupRadix, self).setUp() self.psycopg2_patcher = mock.patch("psycopg2.connect") self.mocked_connection = mock.Mock() self.psycopg2 = self.psycopg2_patcher.start() with self._setup_radix_storage().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): super(TestCleanupRadix, self).tearDown() self.psycopg2_patcher.stop() shutil.rmtree(self.fsrts.config.fs_root) def _setup_radix_storage(self): mock_logging = mock.Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option("logger", default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[{"logger": mock_logging, "minute_slice_interval": 1}], ) return config_manager def _setup_config_manager(self): _super = super(TestCleanupRadix, self)._setup_config_manager config_manager, json_file = _super( "socorro.cron.jobs.cleanup_radix.RadixCleanupCronApp|1d", {"crontabber.class-RadixCleanupCronApp.storage0": {"crashstorage_class": FSDatedRadixTreeStorage}}, ) return config_manager, json_file def test_cleanup_radix(self): self.fsrts._current_slot = lambda: ["00", "00_00"] self.fsrts.save_raw_crash({"test": "TEST"}, {"foo": "bar", self.fsrts.config.dump_field: "baz"}, self.CRASH_ID) self.fsrts._current_slot = lambda: ["10", "00_01"] self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID]) self.assertEqual(list(self.fsrts.new_crashes()), []) config_manager, json_file = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() self.assertEqual(os.listdir(self.fsrts.config.fs_root), []) future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d") future_id = "0bba929f-8721-460c-dead-a43c%s" % future self.fsrts._current_slot = lambda: ["00", "00_00"] self.fsrts.save_raw_crash({"test": "TEST"}, {"foo": "bar", self.fsrts.config.dump_field: "baz"}, future_id) self.fsrts._current_slot = lambda: ["10", "00_01"] self.assertEqual(list(self.fsrts.new_crashes()), [future_id]) self.assertEqual(list(self.fsrts.new_crashes()), []) tab.run_all() self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])
class TestFSDatedRadixTreeStorage(TestCase): CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025" CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026" def setUp(self): super(TestFSDatedRadixTreeStorage, self).setUp() with self._common_config_setup().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): super(TestFSDatedRadixTreeStorage, self).tearDown() shutil.rmtree(self.fsrts.config.fs_root) def _common_config_setup(self): mock_logging = Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1, 'fs_root': os.environ['resource.fs.fs_root'], }], argv_source=[] ) return config_manager def _make_test_crash(self): self.fsrts.save_raw_crash( { # raw crash "test": "TEST" }, MemoryDumpsMapping({ # dumps 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }), self.CRASH_ID_1 ) def test_save_raw_crash(self): try: self._make_test_crash() ok_(os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) ok_(os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) ok_( os.path.exists( "%s/%s.json" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_( os.path.exists( "%s/%s.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_( os.path.exists( "%s/%s.foo.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_save_raw_crash_no_dumps(self): try: self.fsrts.save_raw_crash( { # raw crash "test": "TEST" }, None, # dumps self.CRASH_ID_1 ) ok_(os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) ok_(os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) ok_( os.path.exists( "%s/%s.json" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.foo.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_save_raw_and_processed_crash_no_dumps(self): try: self.fsrts.save_raw_and_processed( { # raw crash "test": "TEST" }, None, # dumps { # processed_crash 'processed': 'crash', 'uuid': self.CRASH_ID_1, }, self.CRASH_ID_1 ) ok_(os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) ok_(os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) ok_( os.path.exists( "%s/%s.json" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_(not os.path.exists( "%s/%s.foo.dump" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) ok_( os.path.exists( "%s/%s.jsonz" % ( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.CRASH_ID_1 ) ) ) finally: self.fsrts.remove(self.CRASH_ID_1) def test_get_raw_crash(self): self._make_test_crash() eq_(self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'], "TEST") assert_raises(CrashIDNotFound, self.fsrts.get_raw_crash, self.CRASH_ID_2) def test_get_raw_dump(self): self._make_test_crash() eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'), "bar") eq_(self.fsrts.get_raw_dump(self.CRASH_ID_1, self.fsrts.config.dump_field), "baz") assert_raises(CrashIDNotFound, self.fsrts.get_raw_dump, self.CRASH_ID_2, "foo") assert_raises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1, "foor") def test_get_raw_dumps(self): self._make_test_crash() eq_(self.fsrts.get_raw_dumps(self.CRASH_ID_1), { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }) assert_raises(CrashIDNotFound, self.fsrts.get_raw_dumps, self.CRASH_ID_2) def test_remove(self): self._make_test_crash() self.fsrts.remove(self.CRASH_ID_1) parent = os.path.realpath( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1))) p = os.path.join(parent, self.CRASH_ID_1) ok_(not os.path.exists(p)) p = os.path.dirname(p) ok_(not os.path.exists(p)) p = os.path.dirname(p) ok_(not os.path.exists(p)) assert_raises(CrashIDNotFound, self.fsrts.remove, self.CRASH_ID_2) def test_new_crashes(self): self.fsrts._current_slot = lambda: ['00', '00_00'] self._make_test_crash() self.fsrts._current_slot = lambda: ['00', '00_01'] eq_(list(self.fsrts.new_crashes()), [self.CRASH_ID_1]) eq_(list(self.fsrts.new_crashes()), []) self.fsrts.remove(self.CRASH_ID_1) del self.fsrts._current_slot
class TestFSDatedRadixTreeStorage(unittest.TestCase): CRASH_ID_1 = "0bba929f-8721-460c-dead-a43c20071025" CRASH_ID_2 = "0bba929f-8721-460c-dead-a43c20071026" def setUp(self): with self._common_config_setup().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): shutil.rmtree(self.fsrts.config.fs_root) def _common_config_setup(self): mock_logging = Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1 }]) return config_manager def _make_test_crash(self): self.fsrts.save_raw_crash({"test": "TEST"}, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, self.CRASH_ID_1) def test_save_raw_crash(self): self._make_test_crash() self.assertTrue( os.path.islink( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1)))) self.assertTrue( os.path.exists( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1), self.CRASH_ID_1))) def test_get_raw_crash(self): self._make_test_crash() self.assertEqual( self.fsrts.get_raw_crash(self.CRASH_ID_1)['test'], "TEST") self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_crash, self.CRASH_ID_2) def test_get_raw_dump(self): self._make_test_crash() self.assertEqual(self.fsrts.get_raw_dump(self.CRASH_ID_1, 'foo'), "bar") self.assertEqual( self.fsrts.get_raw_dump(self.CRASH_ID_1, self.fsrts.config.dump_field), "baz") self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dump, self.CRASH_ID_2, "foo") self.assertRaises(IOError, self.fsrts.get_raw_dump, self.CRASH_ID_1, "foor") def test_get_raw_dumps(self): self._make_test_crash() self.assertEqual(self.fsrts.get_raw_dumps(self.CRASH_ID_1), { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }) self.assertRaises(CrashIDNotFound, self.fsrts.get_raw_dumps, self.CRASH_ID_2) def test_remove(self): self._make_test_crash() self.fsrts.remove(self.CRASH_ID_1) parent = os.path.realpath( os.path.join( self.fsrts._get_radixed_parent_directory(self.CRASH_ID_1), self.fsrts._get_date_root_name(self.CRASH_ID_1))) p = os.path.join(parent, self.CRASH_ID_1) self.assertTrue(not os.path.exists(p)) p = os.path.dirname(p) self.assertTrue(not os.path.exists(p)) p = os.path.dirname(p) self.assertTrue(not os.path.exists(p)) self.assertRaises(CrashIDNotFound, self.fsrts.remove, self.CRASH_ID_2) def test_new_crashes(self): self.fsrts._current_slot = lambda: ['00', '00_00'] self._make_test_crash() self.fsrts._current_slot = lambda: ['00', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID_1]) self.assertEqual(list(self.fsrts.new_crashes()), []) self.fsrts.remove(self.CRASH_ID_1) del self.fsrts._current_slot
class TestCleanupRadix(IntegrationTestCaseBase): CRASH_ID = "0bba929f-8721-460c-dead-a43c20071025" def setUp(self): super(TestCleanupRadix, self).setUp() self.temp_fs_root = tempfile.mkdtemp() with self._setup_radix_storage().context() as config: self.fsrts = FSDatedRadixTreeStorage(config) def tearDown(self): super(TestCleanupRadix, self).tearDown() shutil.rmtree(self.temp_fs_root) def _setup_radix_storage(self): mock_logging = mock.Mock() required_config = FSDatedRadixTreeStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'minute_slice_interval': 1, 'fs_root': self.temp_fs_root }]) return config_manager def _setup_config_manager(self): _super = super(TestCleanupRadix, self)._setup_config_manager return _super( 'socorro.cron.jobs.cleanup_radix.RadixCleanupCronApp|1d', { 'crontabber.class-RadixCleanupCronApp.dated_storage_classes': 'socorro.external.fs.crashstorage.FSDatedRadixTreeStorage', 'fs_root': self.temp_fs_root, }) def test_cleanup_radix(self): self.fsrts._current_slot = lambda: ['00', '00_00'] self.fsrts.save_raw_crash({"test": "TEST"}, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, self.CRASH_ID) self.fsrts._current_slot = lambda: ['10', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID]) self.assertEqual(list(self.fsrts.new_crashes()), []) config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['cleanup_radix'] assert not information['cleanup_radix']['last_error'] assert information['cleanup_radix']['last_success'] self.assertEqual(os.listdir(self.fsrts.config.fs_root), []) future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d") future_id = "0bba929f-8721-460c-dead-a43c%s" % future self.fsrts._current_slot = lambda: ['00', '00_00'] self.fsrts.save_raw_crash({"test": "TEST"}, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, future_id) self.fsrts._current_slot = lambda: ['10', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [future_id]) self.assertEqual(list(self.fsrts.new_crashes()), []) tab.run_all() self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])