def test_unsent_read_purge_within_age(self):
"""Test that when there is unsent data in readings table with user_ts < configured age,
purge process runs but no data is purged
Precondition:
age=72
retainUnsent=False
readings in readings table = 1 with user_ts = now() -15 hours (less than 72)
last_object in streams = 0 (default for all rows)
"""
last_id = self._insert_readings_data(15)
purge = Purge()
purge.run()
log = self._get_log()
assert log[0] == 4
assert log[1]["rowsRemoved"] == 0
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 1
assert log[1]["rowsRemaining"] == 1
stats = self._get_stats()
assert stats[0] == 0
assert stats[1] == 0
readings = self._get_reads()
assert readings["count"] == 1
assert readings["rows"][0]["id"] == last_id
async def test_purge_data_invalid_conf(self, conf, expected_error_key):
"""Test that purge_data raises exception when called with invalid configuration"""
@asyncio.coroutine
def mock_audit_info():
return ""
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
p._logger = logger
p._logger.info = MagicMock()
p._logger.error = MagicMock()
p._storage_async = MagicMock(spec=StorageClientAsync)
p._readings_storage_async = MagicMock(spec=ReadingsStorageClientAsync)
audit = p._audit
with patch.object(p._storage_async, "query_tbl_with_payload",
side_effect=q_result) as patch_storage:
with patch.object(p._readings_storage_async, 'purge', side_effect=self.store_purge) as mock_storage_purge:
with patch.object(audit, 'information', return_value=mock_audit_info()) as audit_info:
# Test the code block when purge failed because of invalid configuration
await p.purge_data(conf)
p._logger.error.assert_called_with('Configuration item {} bla should be integer!'.
format(expected_error_key))
assert patch_storage.called
assert 2 == patch_storage.call_count
def test_unsent_read_purge_old(self):
"""Test that when there is unsent data in readings table with user_ts >= configured age,
purge process runs and data is purged
Precondition:
age=72
retainUnsent=False
readings in readings table = 1 with user_ts = now() - 80 hours
last_object in streams = 0 (default for all rows)
"""
self._insert_readings_data(80)
purge = Purge()
purge.run()
log = self._get_log()
assert log[0] == 4
assert log[1]["rowsRemoved"] == 1
assert log[1]["unsentRowsRemoved"] == 1
assert log[1]["rowsRetained"] == 0
assert log[1]["rowsRemaining"] == 0
stats = self._get_stats()
assert stats[0] == 1
assert stats[1] == 1
readings = self._get_reads()
assert readings["count"] == 0
def test_unsent_read_purge_current(self):
"""Test that when there is unsent data in readings table with user_ts = now,
purge process runs but no data is purged
Precondition:
age=72
retainUnsent=False
readings in readings table = 1 with user_ts = now()
last_object in streams = 0 (default for all rows)
"""
last_id = self._insert_readings_data(0)
purge = Purge("localhost", self._core_management_port)
purge.start()
log = self._get_log()
assert log[0] == 0
assert log[1]["rowsRemoved"] == 0
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 1
assert log[1]["rowsRemaining"] == 1
stats = self._get_stats()
assert stats[0] == 0
assert stats[1] == 0
readings = self._get_reads()
assert readings["count"] == 1
assert readings["rows"][0]["id"] == last_id
async def test_purge_data(self, conf, expected_return, expected_calls):
"""Test that purge_data calls Storage's purge with defined configuration"""
@asyncio.coroutine
def mock_audit_info():
return ""
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
p._logger = logger
p._logger.info = MagicMock()
p._logger.error = MagicMock()
p._storage_async = MagicMock(spec=StorageClientAsync)
p._readings_storage_async = MagicMock(spec=ReadingsStorageClientAsync)
audit = p._audit
with patch.object(p._storage_async, "query_tbl_with_payload",
side_effect=q_result) as patch_storage:
with patch.object(p._readings_storage_async, 'purge', side_effect=self.store_purge) as mock_storage_purge:
with patch.object(audit, 'information', return_value=mock_audit_info()) as audit_info:
# Test the positive case when all if conditions in purge_data pass
assert expected_return == await p.purge_data(conf)
assert audit_info.called
args, kwargs = mock_storage_purge.call_args
assert kwargs == expected_calls
assert patch_storage.called
assert 2 == patch_storage.call_count
async def test_purge_error_storage_response(self, conf, expected_return):
"""Test that purge_data logs error when storage purge returns an error response"""
@asyncio.coroutine
def mock_audit_info():
return ""
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
p._logger = logger
p._logger.info = MagicMock()
p._logger.error = MagicMock()
p._storage_async = MagicMock(spec=StorageClientAsync)
p._readings_storage_async = MagicMock(spec=ReadingsStorageClientAsync)
audit = p._audit
with patch.object(p._storage_async, "query_tbl_with_payload",
side_effect=q_result) as patch_storage:
with patch.object(p._readings_storage_async, 'purge', side_effect=self.store_purge):
with patch.object(audit, 'information', return_value=mock_audit_info()):
assert expected_return == await p.purge_data(conf)
assert patch_storage.called
assert 2 == patch_storage.call_count
def test_all_dest_sent_reads_purge(self):
"""Test that when there is data in readings table which is sent to all historians
with user_ts >= configured age and user_ts = now(),
purge process runs and data is purged
If retainUnsent=False then all readings older than the age passed in,
regardless of the value of sent will be removed
Precondition:
age=72
retainUnsent=False
readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now()
last_object in streams = id of last reading (for all rows)
"""
self._insert_readings_data(80)
last_id = self._insert_readings_data(0)
self._update_streams(rows_to_update=-1, id_last_object=last_id)
purge = Purge()
purge.run()
log = self._get_log()
assert log[0] == 4
assert log[1]["rowsRemoved"] == 1
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 0
assert log[1]["rowsRemaining"] == 1
stats = self._get_stats()
assert stats[0] == 1
assert stats[1] == 0
readings = self._get_reads()
assert readings["count"] == 1
assert readings["rows"][0]["id"] == last_id
def test_unsent_reads_retain(self):
"""Test that when there is unsent data in readings table with user_ts >= configured age and user_ts=now(),
purge process runs and data is purged
Precondition:
age=72
retainUnsent=True
readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now()
last_object in streams = 0 (default for all rows)
"""
self._insert_readings_data(80)
self._insert_readings_data(0)
self._update_configuration(age='72', retain_unsent='True')
purge = Purge()
purge.run()
log = self._get_log()
assert log[0] == 4
assert log[1]["rowsRemoved"] == 0
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 2
assert log[1]["rowsRemaining"] == 2
stats = self._get_stats()
assert stats[0] == 0
assert stats[1] == 0
readings = self._get_reads()
assert readings["count"] == 2
def test_run(self, event_loop):
"""Test that run calls all units of purge process"""
@asyncio.coroutine
def mock_audit_info():
return ""
mockStorageClient = MagicMock(spec=StorageClient)
mockAuditLogger = AuditLogger(mockStorageClient)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge(loop=event_loop)
config = "Some config"
p._logger.exception = MagicMock()
with patch.object(p, 'set_configuration',
return_value=config) as mock_set_config:
with patch.object(p, 'purge_data',
return_value=(1, 2)) as mock_purge_data:
with patch.object(
p, 'write_statistics') as mock_write_stats:
p.run()
# Test the positive case when no error in try block
mock_set_config.assert_called_once_with()
mock_purge_data.assert_called_once_with(config)
mock_write_stats.assert_called_once_with(1, 2)
def test_all_dest_sent_reads_retain(self):
"""Test that when there is data in readings table which is sent to all historians
with user_ts >= configured age and user_ts = now(),
purge process runs and data is purged for only for read where user_ts >= configured age
Precondition:
age=72
retainUnsent=True
readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now()
last_object in streams = id of last reading (for all rows)
"""
self._insert_readings_data(80)
last_id = self._insert_readings_data(0)
self._update_configuration(age='72', retain_unsent='True')
self._update_streams(rows_to_update=-1, id_last_object=last_id)
purge = Purge()
purge.run()
log = self._get_log()
assert log[0] == 4
assert log[1]["rowsRemoved"] == 1
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 0
assert log[1]["rowsRemaining"] == 1
stats = self._get_stats()
assert stats[0] == 1
assert stats[1] == 0
readings = self._get_reads()
assert readings["count"] == 1
assert readings["rows"][0]["id"] == last_id
def test_config_age_purge(self):
"""Test that when there is unsent data in readings table with user_ts < configured age and user_ts=now(),
data older than configured data is deleted
Precondition:
age=10
retainUnsent=False (default)
readings in readings table = 2, one with user_ts = [now() - 15 hours], another with user_ts = now()
last_object in streams = 0 (default for all rows)
"""
self._insert_readings_data(15)
last_id = self._insert_readings_data(0)
self._update_configuration(age='15', retain_unsent='False')
purge = Purge()
purge.run()
log = self._get_log()
assert log[0] == 4
assert log[1]["rowsRemoved"] == 1
assert log[1]["unsentRowsRemoved"] == 1
assert log[1]["rowsRetained"] == 1
assert log[1]["rowsRemaining"] == 1
stats = self._get_stats()
assert stats[0] == 1
assert stats[1] == 1
readings = self._get_reads()
assert readings["count"] == 1
assert readings["rows"][0]["id"] == last_id
def test_one_dest_sent_reads_retain(self):
"""Test that when there is data in readings table which is sent to one historian but not to other
with user_ts >= configured age and user_ts = now(),
purge process runs and data is retained
Precondition:
age=72
retainUnsent=True
readings in readings table = 2, one with user_ts = [now() - 80 hours], another with user_ts = now()
last_object in streams = id of last reading (for one row)
"""
self._insert_readings_data(80)
last_id = self._insert_readings_data(0)
self._update_configuration(age=72, retain_unsent=True)
self._update_streams(rows_to_update=1, id_last_object=last_id)
purge = Purge("localhost", self._core_management_port)
purge.start()
log = self._get_log()
assert log[0] == 0
assert log[1]["rowsRemoved"] == 0
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 2
assert log[1]["rowsRemaining"] == 2
stats = self._get_stats()
assert stats[0] == 0
assert stats[1] == 0
readings = self._get_reads()
assert readings["count"] == 2
def test_no_read_purge(self):
"""Test that when there is no data in readings table, purge process runs but no data is purged"""
purge = Purge()
purge.run()
log = self._get_log()
assert log[0] == 4
assert log[1]["rowsRemoved"] == 0
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 0
assert log[1]["rowsRemaining"] == 0
stats = self._get_stats()
assert stats[0] == 0
assert stats[1] == 0
def test_no_read_purge(self):
"""Test that when there is no data in readings table, purge process runs but no data is purged"""
purge = Purge("localhost", self._core_management_port)
purge.start()
log = self._get_log()
assert log[0] == 0
assert log[1]["rowsRemoved"] == 0
assert log[1]["unsentRowsRemoved"] == 0
assert log[1]["rowsRetained"] == 0
assert log[1]["rowsRemaining"] == 0
stats = self._get_stats()
assert stats[0] == 0
assert stats[1] == 0
async def test_run(self):
"""Test that run calls all units of purge process"""
@asyncio.coroutine
def mock_config():
return "Some config"
@asyncio.coroutine
def mock_purge():
return 1, 2
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
p._logger.exception = MagicMock()
with patch.object(p, 'set_configuration', return_value=mock_config()) as mock_set_config:
with patch.object(p, 'purge_data', return_value=mock_purge()) as mock_purge_data:
with patch.object(p, 'write_statistics') as mock_write_stats:
await p.run()
# Test the positive case when no error in try block
mock_write_stats.assert_called_once_with(1, 2)
mock_purge_data.assert_called_once_with("Some config")
mock_set_config.assert_called_once_with()
async def test_write_statistics(self):
"""Test that write_statistics calls update statistics with defined keys and value increments"""
@asyncio.coroutine
def mock_s_update():
return ""
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, '__init__'):
with patch.object(Statistics, '_load_keys', return_value=mock_s_update()):
with patch.object(Statistics, 'update', return_value=mock_s_update()) as mock_stats_update:
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
p._storage_async = mockStorageClientAsync
await p.write_statistics(1, 2)
mock_stats_update.assert_has_calls([call('PURGED', 1), call('UNSNPURGED', 2)])
async def _purge_instance():
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, "__init__"):
with patch.object(logger, "setup"):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
return p
def test_purge_data_invalid_conf(self, event_loop, conf,
expected_error_key):
"""Test that purge_data raises exception when called with invalid configuration"""
@asyncio.coroutine
def mock_audit_info():
return ""
mockStorageClient = MagicMock(spec=StorageClient)
mockAuditLogger = AuditLogger(mockStorageClient)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge(loop=event_loop)
p._logger = logger
p._logger.info = MagicMock()
p._logger.error = MagicMock()
p._storage = MagicMock(spec=StorageClient)
p._readings_storage = MagicMock(spec=ReadingsStorageClient)
audit = p._audit
with patch.object(p._readings_storage,
'purge',
side_effect=self.store_purge):
with patch.object(audit,
'information',
return_value=mock_audit_info()):
# Test the code block when purge failed because of invalid configuration
p.purge_data(conf)
p._logger.error.assert_called_with(
'Configuration item {} bla should be integer!'.
format(expected_error_key))
def test_init(self):
"""Test that creating an instance of Purge calls init of FoglampProcess and creates loggers"""
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, "__init__") as mock_process:
with patch.object(logger, "setup") as log:
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
assert isinstance(p, Purge)
assert isinstance(p._audit, AuditLogger)
log.assert_called_once_with("Data Purge")
mock_process.assert_called_once_with()
def test_purge_error_storage_response(self, event_loop, conf,
expected_return):
"""Test that purge_data logs error when storage purge returns an error response"""
@asyncio.coroutine
def mock_audit_info():
return ""
mockStorageClient = MagicMock(spec=StorageClient)
mockAuditLogger = AuditLogger(mockStorageClient)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge(loop=event_loop)
p._logger = logger
p._logger.info = MagicMock()
p._logger.error = MagicMock()
p._storage = MagicMock(spec=StorageClient)
p._readings_storage = MagicMock(spec=ReadingsStorageClient)
audit = p._audit
with patch.object(p._readings_storage,
'purge',
side_effect=self.store_purge):
with patch.object(audit,
'information',
return_value=mock_audit_info()):
assert expected_return == p.purge_data(conf)
async def test_set_configuration(self):
"""Test that purge's set_configuration returns configuration item with key 'PURGE_READ' """
@asyncio.coroutine
def mock_cm_return():
return ""
mockStorageClientAsync = MagicMock(spec=StorageClientAsync)
mockAuditLogger = AuditLogger(mockStorageClientAsync)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge()
p._storage = MagicMock(spec=StorageClientAsync)
mock_cm = ConfigurationManager(p._storage)
with patch.object(mock_cm, 'create_category', return_value=mock_cm_return()) as mock_create_cat:
with patch.object(mock_cm, 'get_category_all_items', return_value=mock_cm_return()) \
as mock_get_cat:
await p.set_configuration()
mock_get_cat.assert_called_once_with('PURGE_READ')
args, kwargs = mock_create_cat.call_args
assert len(args) == 3
assert args[0] == 'PURGE_READ'
def test_run_exception(self, event_loop):
"""Test that run calls all units of purge process and checks the exception handling"""
@asyncio.coroutine
def mock_audit_info():
return ""
mockStorageClient = MagicMock(spec=StorageClient)
mockAuditLogger = AuditLogger(mockStorageClient)
with patch.object(FoglampProcess, '__init__'):
with patch.object(mockAuditLogger, "__init__", return_value=None):
p = Purge(loop=event_loop)
config = "Some config"
p._logger.exception = MagicMock()
with patch.object(p, 'set_configuration', return_value=config):
with patch.object(p,
'purge_data',
return_value=Exception()):
with patch.object(p, 'write_statistics'):
p.run()
# Test the negative case when function purge_data raise some exception
p._logger.exception.assert_called_once_with(
"'Exception' object is not iterable")
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END
"""Purge process starter"""
from foglamp.tasks.purge.purge import Purge
from foglamp.common import logger
__author__ = "Terris Linenbach, Vaibhav Singhal"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"
if __name__ == '__main__':
_logger = logger.setup("Purge")
purge_process = Purge()
purge_process.run()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END
"""Purge process starter"""
import asyncio
from foglamp.tasks.purge.purge import Purge
from foglamp.common import logger
__author__ = "Terris Linenbach, Vaibhav Singhal"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"
if __name__ == '__main__':
_logger = logger.setup("Purge")
loop = asyncio.get_event_loop()
purge_process = Purge()
loop.run_until_complete(purge_process.run())
