def _cleanup():
"""Do cleanup."""
while True:
zk.prune_trace_evictions(
context.GLOBAL.zk.conn,
trace_evictions_max_count
)
zk.prune_trace_service_events(
context.GLOBAL.zk.conn,
trace_service_events_max_count
)
zk.cleanup_trace(
context.GLOBAL.zk.conn,
trace_batch_size,
trace_expire_after
)
zk.cleanup_finished(
context.GLOBAL.zk.conn,
finished_batch_size,
finished_expire_after
)
zk.cleanup_trace_history(
context.GLOBAL.zk.conn,
trace_history_max_count
)
zk.cleanup_finished_history(
context.GLOBAL.zk.conn,
finished_history_max_count
)
_LOGGER.info('Finished cleanup, sleep %s sec', interval)
time.sleep(interval)
def test_finished_cleanup(self):
"""Tests tasks cleanup.
"""
data = b"{data: '1.0', host: foo, state: finished, when: '123.45'}\n"
zk_content = {
'trace': {
'0001': {},
'0002': {},
},
'finished': {
'app1#0001': {
'.metadata': {
'last_modified': 1000
},
'.data': data
},
'app1#0002': {
'.metadata': {
'last_modified': 1000
},
'.data': data
},
'app1#0003': {
'.metadata': {
'last_modified': 1000
},
'.data': data
},
'app1#0004': {
'.metadata': {
'last_modified': 1000
},
'.data': data
},
'app1#0005': {
'.metadata': {
'last_modified': 1000
},
'.data': data
},
'app1#0006': {
'.metadata': {
'last_modified': 1000
},
'.data': data
},
'app1#0007': {
'.metadata': {
'last_modified': 1000
},
'.data': data
},
},
'trace.history': {},
'finished.history': {},
}
self.make_mock_zk(zk_content)
zkclient = treadmill.zkutils.ZkClient()
conn_mock = mock.MagicMock()
sqlite3.connect.return_value = conn_mock
zk.cleanup_finished(zkclient, 10, 3)
self.assertFalse(kazoo.client.KazooClient.create.called)
# Current time - 1000, expiration - 3 seconds, there are < 10 events
# that are expired, nothing is uploaded.
self.assertEqual(0, len(zk_content['finished.history']))
time.time.return_value = 1100
# There are twelve expired events, expect batch to be uploaded
zk.cleanup_finished(zkclient, 5, 3)
conn_mock.executemany.assert_called_once_with(
"""
INSERT INTO finished (
path, timestamp, data, directory, name
) VALUES(?, ?, ?, ?, ?)
""",
[('/finished/app1#0001', 1000.0,
"{data: '1.0', host: foo, state: finished, when: '123.45'}\n",
'/finished', 'app1#0001'),
('/finished/app1#0002', 1000.0,
"{data: '1.0', host: foo, state: finished, when: '123.45'}\n",
'/finished', 'app1#0002'),
('/finished/app1#0003', 1000.0,
"{data: '1.0', host: foo, state: finished, when: '123.45'}\n",
'/finished', 'app1#0003'),
('/finished/app1#0004', 1000.0,
"{data: '1.0', host: foo, state: finished, when: '123.45'}\n",
'/finished', 'app1#0004'),
('/finished/app1#0005', 1000.0,
"{data: '1.0', host: foo, state: finished, when: '123.45'}\n",
'/finished', 'app1#0005')])
kazoo.client.KazooClient.create.assert_called_with(
'/finished.history/finished.db.gzip-',
value=mock.ANY,
acl=mock.ANY,
makepath=True,
ephemeral=False,
sequence=True,
)
self.assertEqual(5, kazoo.client.KazooClient.delete.call_count)
def test_finished_cleanup(self):
""""Tests tasks cleanup."""
zk_content = {
'trace': {
'0001': {},
'0002': {},
},
'finished': {
'app1#0001': {
'.metadata': {
'last_modified': 1000
}
},
'app1#0002': {
'.metadata': {
'last_modified': 1000
}
},
'app1#0003': {
'.metadata': {
'last_modified': 1000
}
},
'app1#0004': {
'.metadata': {
'last_modified': 1000
}
},
'app1#0005': {
'.metadata': {
'last_modified': 1000
}
},
'app1#0006': {
'.metadata': {
'last_modified': 1000
}
},
'app1#0007': {
'.metadata': {
'last_modified': 1000
}
},
},
'trace.history': {},
'finished.history': {},
}
self.make_mock_zk(zk_content)
zkclient = kazoo.client.KazooClient()
zk.cleanup_finished(zkclient, 10, 3)
self.assertFalse(kazoo.client.KazooClient.create.called)
# Current time - 1000, expiration - 3 seconds, there are < 10 events
# that are expired, nothing is uploaded.
self.assertEqual(0, len(zk_content['finished.history']))
time.time.return_value = 1100
# There are twelve expired events, expect batch to be uploaded
zk.cleanup_finished(zkclient, 5, 3)
kazoo.client.KazooClient.create.assert_called_with(
'/finished.history/finished.db.gzip-',
mock.ANY,
acl=mock.ANY,
makepath=True,
ephemeral=False,
sequence=True,
)
self.assertEqual(5, kazoo.client.KazooClient.delete.call_count)