async def test_purge_error_storage_response(self, conf, expected_return): """Test that purge_data logs error when storage purge returns an error response""" @asyncio.coroutine def mock_audit_info(): return "" mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() p._logger = logger p._logger.info = MagicMock() p._logger.error = MagicMock() p._storage_async = MagicMock(spec=StorageClientAsync) p._readings_storage_async = MagicMock(spec=ReadingsStorageClientAsync) audit = p._audit with patch.object(p._storage_async, "query_tbl_with_payload", side_effect=q_result) as patch_storage: with patch.object(p._readings_storage_async, 'purge', side_effect=self.store_purge): with patch.object(audit, 'information', return_value=mock_audit_info()): assert expected_return == await p.purge_data(conf) assert patch_storage.called assert 2 == patch_storage.call_count
def test_all_dest_sent_reads_purge(self): """Test that when there is data in readings table which is sent to all historians with user_ts >= configured age and user_ts = now(), purge process runs and data is purged If retainUnsent=False then all readings older than the age passed in, regardless of the value of sent will be removed Precondition: age=72 retainUnsent=False readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now() last_object in streams = id of last reading (for all rows) """ self._insert_readings_data(80) last_id = self._insert_readings_data(0) self._update_streams(rows_to_update=-1, id_last_object=last_id) purge = Purge() purge.run() log = self._get_log() assert log[0] == 4 assert log[1]["rowsRemoved"] == 1 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 0 assert log[1]["rowsRemaining"] == 1 stats = self._get_stats() assert stats[0] == 1 assert stats[1] == 0 readings = self._get_reads() assert readings["count"] == 1 assert readings["rows"][0]["id"] == last_id
def test_config_age_purge(self): """Test that when there is unsent data in readings table with user_ts < configured age and user_ts=now(), data older than configured data is deleted Precondition: age=10 retainUnsent=False (default) readings in readings table = 2, one with user_ts = [now() - 15 hours], another with user_ts = now() last_object in streams = 0 (default for all rows) """ self._insert_readings_data(15) last_id = self._insert_readings_data(0) self._update_configuration(age='15', retain_unsent='False') purge = Purge() purge.run() log = self._get_log() assert log[0] == 4 assert log[1]["rowsRemoved"] == 1 assert log[1]["unsentRowsRemoved"] == 1 assert log[1]["rowsRetained"] == 1 assert log[1]["rowsRemaining"] == 1 stats = self._get_stats() assert stats[0] == 1 assert stats[1] == 1 readings = self._get_reads() assert readings["count"] == 1 assert readings["rows"][0]["id"] == last_id
async def test_purge_data(self, conf, expected_return, expected_calls): """Test that purge_data calls Storage's purge with defined configuration""" @asyncio.coroutine def mock_audit_info(): return "" mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() p._logger = logger p._logger.info = MagicMock() p._logger.error = MagicMock() p._storage_async = MagicMock(spec=StorageClientAsync) p._readings_storage_async = MagicMock(spec=ReadingsStorageClientAsync) audit = p._audit with patch.object(p._storage_async, "query_tbl_with_payload", side_effect=q_result) as patch_storage: with patch.object(p._readings_storage_async, 'purge', side_effect=self.store_purge) as mock_storage_purge: with patch.object(audit, 'information', return_value=mock_audit_info()) as audit_info: # Test the positive case when all if conditions in purge_data pass assert expected_return == await p.purge_data(conf) assert audit_info.called args, kwargs = mock_storage_purge.call_args assert kwargs == expected_calls assert patch_storage.called assert 2 == patch_storage.call_count
def test_purge_data_invalid_conf(self, event_loop, conf, expected_error_key): """Test that purge_data raises exception when called with invalid configuration""" @asyncio.coroutine def mock_audit_info(): return "" mockStorageClient = MagicMock(spec=StorageClient) mockAuditLogger = AuditLogger(mockStorageClient) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge(loop=event_loop) p._logger = logger p._logger.info = MagicMock() p._logger.error = MagicMock() p._storage = MagicMock(spec=StorageClient) p._readings_storage = MagicMock(spec=ReadingsStorageClient) audit = p._audit with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge): with patch.object(audit, 'information', return_value=mock_audit_info()): # Test the code block when purge failed because of invalid configuration p.purge_data(conf) p._logger.error.assert_called_with( 'Configuration item {} bla should be integer!'. format(expected_error_key))
def test_purge_error_storage_response(self, event_loop, conf, expected_return): """Test that purge_data logs error when storage purge returns an error response""" @asyncio.coroutine def mock_audit_info(): return "" mockStorageClient = MagicMock(spec=StorageClient) mockAuditLogger = AuditLogger(mockStorageClient) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge(loop=event_loop) p._logger = logger p._logger.info = MagicMock() p._logger.error = MagicMock() p._storage = MagicMock(spec=StorageClient) p._readings_storage = MagicMock(spec=ReadingsStorageClient) audit = p._audit with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge): with patch.object(audit, 'information', return_value=mock_audit_info()): assert expected_return == p.purge_data(conf)
def test_one_dest_sent_reads_retain(self): """Test that when there is data in readings table which is sent to one historian but not to other with user_ts >= configured age and user_ts = now(), purge process runs and data is retained Precondition: age=72 retainUnsent=True readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now() last_object in streams = id of last reading (for one row) """ self._insert_readings_data(80) last_id = self._insert_readings_data(0) self._update_configuration(age=72, retain_unsent=True) self._update_streams(rows_to_update=1, id_last_object=last_id) purge = Purge("localhost", self._core_management_port) purge.start() log = self._get_log() assert log[0] == 0 assert log[1]["rowsRemoved"] == 0 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 2 assert log[1]["rowsRemaining"] == 2 stats = self._get_stats() assert stats[0] == 0 assert stats[1] == 0 readings = self._get_reads() assert readings["count"] == 2
async def test_run(self): """Test that run calls all units of purge process""" @asyncio.coroutine def mock_audit_info(): return "" @asyncio.coroutine def mock_config(): return "Some config" @asyncio.coroutine def mock_purge(): return 1, 2 mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() p._logger.exception = MagicMock() with patch.object(p, 'set_configuration', return_value=mock_config()) as mock_set_config: with patch.object(p, 'purge_data', return_value=mock_purge()) as mock_purge_data: with patch.object(p, 'write_statistics') as mock_write_stats: await p.run() # Test the positive case when no error in try block mock_write_stats.assert_called_once_with(1, 2) mock_purge_data.assert_called_once_with("Some config") mock_set_config.assert_called_once_with()
async def test_purge_data_invalid_conf(self, conf, expected_error_key): """Test that purge_data raises exception when called with invalid configuration""" @asyncio.coroutine def mock_audit_info(): return "" mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() p._logger = logger p._logger.info = MagicMock() p._logger.error = MagicMock() p._storage_async = MagicMock(spec=StorageClientAsync) p._readings_storage_async = MagicMock(spec=ReadingsStorageClientAsync) audit = p._audit with patch.object(p._storage_async, "query_tbl_with_payload", side_effect=q_result) as patch_storage: with patch.object(p._readings_storage_async, 'purge', side_effect=self.store_purge) as mock_storage_purge: with patch.object(audit, 'information', return_value=mock_audit_info()) as audit_info: # Test the code block when purge failed because of invalid configuration await p.purge_data(conf) p._logger.error.assert_called_with('Configuration item {} bla should be integer!'. format(expected_error_key)) assert patch_storage.called assert 2 == patch_storage.call_count
def test_unsent_read_purge_old(self): """Test that when there is unsent data in readings table with user_ts >= configured age, purge process runs and data is purged Precondition: age=72 retainUnsent=False readings in readings table = 1 with user_ts = now() - 80 hours last_object in streams = 0 (default for all rows) """ self._insert_readings_data(80) purge = Purge() purge.run() log = self._get_log() assert log[0] == 4 assert log[1]["rowsRemoved"] == 1 assert log[1]["unsentRowsRemoved"] == 1 assert log[1]["rowsRetained"] == 0 assert log[1]["rowsRemaining"] == 0 stats = self._get_stats() assert stats[0] == 1 assert stats[1] == 1 readings = self._get_reads() assert readings["count"] == 0
def test_unsent_read_purge_within_age(self): """Test that when there is unsent data in readings table with user_ts < configured age, purge process runs but no data is purged Precondition: age=72 retainUnsent=False readings in readings table = 1 with user_ts = now() -15 hours (less than 72) last_object in streams = 0 (default for all rows) """ last_id = self._insert_readings_data(15) purge = Purge() purge.run() log = self._get_log() assert log[0] == 4 assert log[1]["rowsRemoved"] == 0 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 1 assert log[1]["rowsRemaining"] == 1 stats = self._get_stats() assert stats[0] == 0 assert stats[1] == 0 readings = self._get_reads() assert readings["count"] == 1 assert readings["rows"][0]["id"] == last_id
def test_unsent_reads_retain(self): """Test that when there is unsent data in readings table with user_ts >= configured age and user_ts=now(), purge process runs and data is purged Precondition: age=72 retainUnsent=True readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now() last_object in streams = 0 (default for all rows) """ self._insert_readings_data(80) self._insert_readings_data(0) self._update_configuration(age='72', retain_unsent='True') purge = Purge() purge.run() log = self._get_log() assert log[0] == 4 assert log[1]["rowsRemoved"] == 0 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 2 assert log[1]["rowsRemaining"] == 2 stats = self._get_stats() assert stats[0] == 0 assert stats[1] == 0 readings = self._get_reads() assert readings["count"] == 2
def test_all_dest_sent_reads_retain(self): """Test that when there is data in readings table which is sent to all historians with user_ts >= configured age and user_ts = now(), purge process runs and data is purged for only for read where user_ts >= configured age Precondition: age=72 retainUnsent=True readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now() last_object in streams = id of last reading (for all rows) """ self._insert_readings_data(80) last_id = self._insert_readings_data(0) self._update_configuration(age='72', retain_unsent='True') self._update_streams(rows_to_update=-1, id_last_object=last_id) purge = Purge() purge.run() log = self._get_log() assert log[0] == 4 assert log[1]["rowsRemoved"] == 1 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 0 assert log[1]["rowsRemaining"] == 1 stats = self._get_stats() assert stats[0] == 1 assert stats[1] == 0 readings = self._get_reads() assert readings["count"] == 1 assert readings["rows"][0]["id"] == last_id
def test_unsent_read_purge_current(self): """Test that when there is unsent data in readings table with user_ts = now, purge process runs but no data is purged Precondition: age=72 retainUnsent=False readings in readings table = 1 with user_ts = now() last_object in streams = 0 (default for all rows) """ last_id = self._insert_readings_data(0) purge = Purge("localhost", self._core_management_port) purge.start() log = self._get_log() assert log[0] == 0 assert log[1]["rowsRemoved"] == 0 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 1 assert log[1]["rowsRemaining"] == 1 stats = self._get_stats() assert stats[0] == 0 assert stats[1] == 0 readings = self._get_reads() assert readings["count"] == 1 assert readings["rows"][0]["id"] == last_id
async def _purge_instance(): mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, "__init__"): with patch.object(logger, "setup"): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() return p
def test_init(self): """Test that creating an instance of Purge calls init of FoglampProcess and creates loggers""" mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, "__init__") as mock_process: with patch.object(logger, "setup") as log: with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() assert isinstance(p, Purge) assert isinstance(p._audit, AuditLogger) log.assert_called_once_with("Data Purge") mock_process.assert_called_once_with()
def test_no_read_purge(self): """Test that when there is no data in readings table, purge process runs but no data is purged""" purge = Purge() purge.run() log = self._get_log() assert log[0] == 4 assert log[1]["rowsRemoved"] == 0 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 0 assert log[1]["rowsRemaining"] == 0 stats = self._get_stats() assert stats[0] == 0 assert stats[1] == 0
def test_no_read_purge(self): """Test that when there is no data in readings table, purge process runs but no data is purged""" purge = Purge("localhost", self._core_management_port) purge.start() log = self._get_log() assert log[0] == 0 assert log[1]["rowsRemoved"] == 0 assert log[1]["unsentRowsRemoved"] == 0 assert log[1]["rowsRetained"] == 0 assert log[1]["rowsRemaining"] == 0 stats = self._get_stats() assert stats[0] == 0 assert stats[1] == 0
async def test_write_statistics(self): """Test that write_statistics calls update statistics with defined keys and value increments""" @asyncio.coroutine def mock_s_update(): return "" mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, '__init__'): with patch.object(Statistics, '_load_keys', return_value=mock_s_update()): with patch.object(Statistics, 'update', return_value=mock_s_update()) as mock_stats_update: with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() p._storage_async = mockStorageClientAsync await p.write_statistics(1, 2) mock_stats_update.assert_has_calls([call('PURGED', 1), call('UNSNPURGED', 2)])
async def test_set_configuration(self): """Test that purge's set_configuration returns configuration item with key 'PURGE_READ' """ @asyncio.coroutine def mock_cm_return(): return "" mockStorageClientAsync = MagicMock(spec=StorageClientAsync) mockAuditLogger = AuditLogger(mockStorageClientAsync) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge() p._storage = MagicMock(spec=StorageClientAsync) mock_cm = ConfigurationManager(p._storage) with patch.object(mock_cm, 'create_category', return_value=mock_cm_return()) as mock_create_cat: with patch.object(mock_cm, 'get_category_all_items', return_value=mock_cm_return()) \ as mock_get_cat: await p.set_configuration() mock_get_cat.assert_called_once_with('PURGE_READ') args, kwargs = mock_create_cat.call_args assert len(args) == 3 assert args[0] == 'PURGE_READ'
def test_run_exception(self, event_loop): """Test that run calls all units of purge process and checks the exception handling""" @asyncio.coroutine def mock_audit_info(): return "" mockStorageClient = MagicMock(spec=StorageClient) mockAuditLogger = AuditLogger(mockStorageClient) with patch.object(FoglampProcess, '__init__'): with patch.object(mockAuditLogger, "__init__", return_value=None): p = Purge(loop=event_loop) config = "Some config" p._logger.exception = MagicMock() with patch.object(p, 'set_configuration', return_value=config): with patch.object(p, 'purge_data', return_value=Exception()): with patch.object(p, 'write_statistics'): p.run() # Test the negative case when function purge_data raise some exception p._logger.exception.assert_called_once_with( "'Exception' object is not iterable")
#!/usr/bin/env python3 # -*- coding: utf-8 -*- # FOGLAMP_BEGIN # See: http://foglamp.readthedocs.io/ # FOGLAMP_END """Purge process starter""" import asyncio from foglamp.tasks.purge.purge import Purge from foglamp.common import logger __author__ = "Terris Linenbach, Vaibhav Singhal" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" __license__ = "Apache 2.0" __version__ = "${VERSION}" if __name__ == '__main__': _logger = logger.setup("Purge") loop = asyncio.get_event_loop() purge_process = Purge() loop.run_until_complete(purge_process.run())