def _cleanup():
"""Do cleanup."""
while True:
zk.prune_trace_evictions(
context.GLOBAL.zk.conn,
trace_evictions_max_count
)
zk.prune_trace_service_events(
context.GLOBAL.zk.conn,
trace_service_events_max_count
)
zk.cleanup_trace(
context.GLOBAL.zk.conn,
trace_batch_size,
trace_expire_after
)
zk.cleanup_finished(
context.GLOBAL.zk.conn,
finished_batch_size,
finished_expire_after
)
zk.cleanup_trace_history(
context.GLOBAL.zk.conn,
trace_history_max_count
)
zk.cleanup_finished_history(
context.GLOBAL.zk.conn,
finished_history_max_count
)
_LOGGER.info('Finished cleanup, sleep %s sec', interval)
time.sleep(interval)
def test_trace_cleanup(self):
""""Tests tasks cleanup."""
zk_content = {
'trace': {
'0001': {
'app1#0001,1000.00,s1,configured,2DqcoXnaIXEgy': {},
'app1#0001,1001.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1003.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1004.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1005.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1006.00,configured,2DqcoXnaIXEgy': {},
},
'0002': {
'app1#0002,1000.00,s1,configured,2DqcoXnaIXEgy': {},
'app1#0002,1001.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1003.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1004.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1005.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1006.00,configured,2DqcoXnaIXEgy': {},
},
},
'finished': {},
'trace.history': {},
'finished.history': {},
}
self.make_mock_zk(zk_content)
zkclient = kazoo.client.KazooClient()
zk.cleanup_trace(zkclient, 10, 3)
self.assertFalse(kazoo.client.KazooClient.create.called)
# Current time - 1000, expiration - 3 seconds, there are < 10 events
# that are expired, nothing is uploaded.
self.assertEqual(0, len(zk_content['trace.history']))
time.time.return_value = 1100
# There are twelve expired events, expect batch to be uploaded
zk.cleanup_trace(zkclient, 10, 3)
kazoo.client.KazooClient.create.assert_called_with(
'/trace.history/trace.db.gzip-',
mock.ANY,
acl=mock.ANY,
makepath=True,
ephemeral=False,
sequence=True,
)
self.assertEqual(10, kazoo.client.KazooClient.delete.call_count)
def test_trace_cleanup(self):
"""Tests tasks cleanup.
"""
zk_content = {
'scheduled': {
'app1#0003': {},
'app1#0004': {},
},
'trace': {
'0001': {
'app1#0001,1000.00,s1,configured,2DqcoXnaIXEgy': {},
'app1#0001,1001.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1003.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1004.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1005.00,configured,2DqcoXnaIXEgy': {},
'app1#0001,1006.00,configured,2DqcoXnaIXEgy': {},
},
'0002': {
'app1#0002,1000.00,s1,configured,2DqcoXnaIXEgy': {},
'app1#0002,1001.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1003.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1004.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1005.00,configured,2DqcoXnaIXEgy': {},
'app1#0002,1006.00,configured,2DqcoXnaIXEgy': {},
},
'0003': {
'app1#0003,1000.00,s1,configured,2DqcoXnaIXEgy': {},
'app1#0003,1001.00,configured,2DqcoXnaIXEgy': {},
'app1#0003,1003.00,configured,2DqcoXnaIXEgy': {},
'app1#0003,1004.00,configured,2DqcoXnaIXEgy': {},
'app1#0003,1005.00,configured,2DqcoXnaIXEgy': {},
'app1#0003,1006.00,configured,2DqcoXnaIXEgy': {},
},
'0004': {
'app1#0004,1000.00,s1,configured,2DqcoXnaIXEgy': {},
'app1#0004,1001.00,configured,2DqcoXnaIXEgy': {},
'app1#0004,1003.00,configured,2DqcoXnaIXEgy': {},
'app1#0004,1004.00,configured,2DqcoXnaIXEgy': {},
'app1#0004,1005.00,configured,2DqcoXnaIXEgy': {},
'app1#0004,1006.00,configured,2DqcoXnaIXEgy': {},
},
},
'finished': {},
'trace.history': {},
'finished.history': {},
}
self.make_mock_zk(zk_content)
zkclient = treadmill.zkutils.ZkClient()
conn_mock = mock.MagicMock()
sqlite3.connect.return_value = conn_mock
# Current time - 1000, expiration - 3 seconds, there are < 10 events
# that are expired, nothing is uploaded.
zk.cleanup_trace(zkclient, 10, 3)
self.assertEqual(0, len(zk_content['trace.history']))
self.assertFalse(kazoo.client.KazooClient.create.called)
# There are twelve expired events, expect batch to be uploaded.
# Instances app1#0003 and 0004 are running and will not be included.
time.time.return_value = 1100
zk.cleanup_trace(zkclient, 10, 3)
conn_mock.executemany.assert_called_once_with(
"""
INSERT INTO trace (
path, timestamp, data, directory, name
) VALUES(?, ?, ?, ?, ?)
""", [('/trace/0001/app1#0001,1000.00,s1,configured,2DqcoXnaIXEgy',
1000.0, None, '/trace/0001',
'app1#0001,1000.00,s1,configured,2DqcoXnaIXEgy'),
('/trace/0002/app1#0002,1000.00,s1,configured,2DqcoXnaIXEgy',
1000.0, None, '/trace/0002',
'app1#0002,1000.00,s1,configured,2DqcoXnaIXEgy'),
('/trace/0001/app1#0001,1001.00,configured,2DqcoXnaIXEgy',
1001.0, None, '/trace/0001',
'app1#0001,1001.00,configured,2DqcoXnaIXEgy'),
('/trace/0002/app1#0002,1001.00,configured,2DqcoXnaIXEgy',
1001.0, None, '/trace/0002',
'app1#0002,1001.00,configured,2DqcoXnaIXEgy'),
('/trace/0001/app1#0001,1003.00,configured,2DqcoXnaIXEgy',
1003.0, None, '/trace/0001',
'app1#0001,1003.00,configured,2DqcoXnaIXEgy'),
('/trace/0002/app1#0002,1003.00,configured,2DqcoXnaIXEgy',
1003.0, None, '/trace/0002',
'app1#0002,1003.00,configured,2DqcoXnaIXEgy'),
('/trace/0001/app1#0001,1004.00,configured,2DqcoXnaIXEgy',
1004.0, None, '/trace/0001',
'app1#0001,1004.00,configured,2DqcoXnaIXEgy'),
('/trace/0002/app1#0002,1004.00,configured,2DqcoXnaIXEgy',
1004.0, None, '/trace/0002',
'app1#0002,1004.00,configured,2DqcoXnaIXEgy'),
('/trace/0001/app1#0001,1005.00,configured,2DqcoXnaIXEgy',
1005.0, None, '/trace/0001',
'app1#0001,1005.00,configured,2DqcoXnaIXEgy'),
('/trace/0002/app1#0002,1005.00,configured,2DqcoXnaIXEgy',
1005.0, None, '/trace/0002',
'app1#0002,1005.00,configured,2DqcoXnaIXEgy')])
kazoo.client.KazooClient.create.assert_called_with(
'/trace.history/trace.db.gzip-',
value=mock.ANY,
acl=mock.ANY,
makepath=True,
ephemeral=False,
sequence=True,
)
self.assertEqual(10, kazoo.client.KazooClient.delete.call_count)
self.assertEqual(kazoo.client.KazooClient.delete.call_args_list, [
(('/trace/0001/app1#0001,1000.00,s1,configured,2DqcoXnaIXEgy', ),
),
(('/trace/0002/app1#0002,1000.00,s1,configured,2DqcoXnaIXEgy', ),
),
(('/trace/0001/app1#0001,1001.00,configured,2DqcoXnaIXEgy', ), ),
(('/trace/0002/app1#0002,1001.00,configured,2DqcoXnaIXEgy', ), ),
(('/trace/0001/app1#0001,1003.00,configured,2DqcoXnaIXEgy', ), ),
(('/trace/0002/app1#0002,1003.00,configured,2DqcoXnaIXEgy', ), ),
(('/trace/0001/app1#0001,1004.00,configured,2DqcoXnaIXEgy', ), ),
(('/trace/0002/app1#0002,1004.00,configured,2DqcoXnaIXEgy', ), ),
(('/trace/0001/app1#0001,1005.00,configured,2DqcoXnaIXEgy', ), ),
(('/trace/0002/app1#0002,1005.00,configured,2DqcoXnaIXEgy', ), ),
])