def test_task_cleanup(self):
""""Tests expired tasks removal."""
zk_content = {
'scheduled': {
'app1#003': '',
},
'tasks': {
'app1': {
'001': {
# Old task, to be removed.
'xxx-001': {
'.metadata': {
'last_modified': time.time() - 101
}
},
'yyy-002': {
'.metadata': {
'last_modified': time.time() - 101
}
},
},
# Task still active, as not all subnodes expired.
'002': {
'xxx-001': {
'.metadata': {
'last_modified': time.time() - 101
}
},
'yyy-002': {
'.metadata': {
'last_modified': time.time() - 10
}
}
},
# Active task is not evaluated.
'003': {}
},
}
}
self.make_mock_zk(zk_content)
zkclient = kazoo.client.KazooClient()
zk.cleanup(zkclient, 100, max_events=1)
kazoo.client.KazooClient.delete.assert_has_calls(
[
# 002 task has > 1 event, extra will be removed.
mock.call('/tasks/app1/002/xxx-001'),
# 001 task has > 1 event, extra will be removed.
mock.call('/tasks/app1/001/xxx-001'),
# 001 task is expired, recursive delete.
mock.call('/tasks/app1/001', recursive=True),
# try to delete (and fail as not empty)
mock.call('/tasks/app1'),
],
any_order=True)
def cleanup(expiration, interval):
"""Cleans up old tasks."""
context.GLOBAL.zk.conn.ensure_path('/task-cleanup-election')
me = '%s' % (sysinfo.hostname())
lock = context.GLOBAL.zk.conn.Lock('/task-cleanup-election', me)
_LOGGER.info('Waiting for leader lock.')
with lock:
while True:
zk.cleanup(context.GLOBAL.zk.conn, expiration)
_LOGGER.info('Finished cleanup, sleep %s sec', interval)
time.sleep(interval)
def test_no_tasks_history_cleanup(self):
""""Tests tasks history cleanup when there is nothing to delete yet."""
zk_content = {
'tasks': {},
'tasks.history': {
'tasks.db.gzip-0000000001': ''
}
}
self.make_mock_zk(zk_content)
zkclient = kazoo.client.KazooClient()
zk.cleanup(zkclient)
self.assertEquals(treadmill.zkutils.ensure_deleted.call_count, 0)
def test_tasks_history_cleanup(self):
""""Tests tasks history cleanup."""
zk_content = {
'tasks': {},
'tasks.history': dict(
('tasks.db.gzip-%010d' % i, '') for i in range(1, 1003)
)
}
self.make_mock_zk(zk_content)
zkclient = kazoo.client.KazooClient()
zk.cleanup(zkclient)
self.assertEquals(
treadmill.zkutils.ensure_deleted.call_args_list,
[
mock.call(
zkclient, '/tasks.history/tasks.db.gzip-0000000001', False
),
mock.call(
zkclient, '/tasks.history/tasks.db.gzip-0000000002', False
)
]
)
def test_tasks_cleanup(self):
""""Tests tasks cleanup."""
zk_content = {
'scheduled': {
'app1#0000000001': '',
'app2#0000000338': '',
'app2#0000000339': ''
},
'tasks': {
'app1': {
'0000000001': {}
},
'app2': {
'0000000337': {
'.data': ("{data: '256.256', host: host,"
" state: finished, when: '1.2'}\n"),
'.metadata': {
'last_modified': 2.2
},
'1.1,host,service_exited,abc0.test.0.0': '',
'1.2,host,finished,0.0': ''
},
'0000000338': {},
'0000000339': {},
}
},
'tasks.history': {}
}
# Generate 335 finished tasks for app1 (this is > 1000 rows in total)
for i in range(2, 337):
zk_content['tasks']['app1']['%010d' % i] = {
'.data': ("{data: '0.0', host: host,"
" state: finished, when: '1.2'}\n"),
'.metadata': {
'last_modified': 2.2
},
'1.1,host,service_exited,abc0.test.0.0': '',
'1.2,host,finished,0.0': ''
}
self.make_mock_zk(zk_content)
zkclient = kazoo.client.KazooClient()
zk.cleanup(zkclient)
rows1 = []
for i in range(2, 336):
rows1.extend([
('/tasks/app1/%010d' % i, 2,
"{data: '0.0', host: host, state: finished, when: '1.2'}\n"),
('/tasks/app1/%010d/1.1,host,service_exited,abc0.test.0.0' % i,
1, None),
('/tasks/app1/%010d/1.2,host,finished,0.0' % i,
1, None)
])
# Second commit, last finished app1 task and finished app2 task
rows2 = [
('/tasks/app1/0000000336', 2,
"{data: '0.0', host: host, state: finished, when: '1.2'}\n"),
('/tasks/app1/0000000336/1.1,host,service_exited,abc0.test.0.0',
1, None),
('/tasks/app1/0000000336/1.2,host,finished,0.0',
1, None),
('/tasks/app2/0000000337', 2,
"{data: '256.256', host: host, state: finished, when: '1.2'}\n"),
('/tasks/app2/0000000337/1.1,host,service_exited,abc0.test.0.0',
1, None),
('/tasks/app2/0000000337/1.2,host,finished,0.0',
1, None)
]
self.assertEquals(
treadmill.apptrace.zk.TaskDB.add.call_args_list,
[
mock.call(rows1),
mock.call(rows2, close=True)
]
)
self.assertEquals(zk_content, {
'scheduled': {
'app1#0000000001': '',
'app2#0000000338': '',
'app2#0000000339': ''
},
'tasks': {
'app1': {
'0000000001': {}
},
'app2': {
'0000000338': {},
'0000000339': {}
}
},
'tasks.history': {}
})
def _cleanup():
"""Do cleanup."""
while True:
zk.cleanup(context.GLOBAL.zk.conn)
_LOGGER.info('Finished cleanup, sleep %s sec', interval)
time.sleep(interval)