def test_push_noop(self):
"""Test pushing data from the cache to Zookeeper.
"""
# pylint: disable=protected-access
zk_content = {
'zk': {
'path': {
'FOO#chk2#0000000042': b'foo',
}
}
}
self.make_mock_zk(zk_content)
zkclient = treadmill.zkutils.ZkClient()
zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir)
zdc._cached = {
'FOO': [
zkdatacache.ZkCachedEntry(
fname='a#chk2', chksum='chk2', ctime='0'
),
zkdatacache.ZkCachedEntry(
fname='a#chk1', chksum='chk1', ctime='0'
),
]
}
# Pushing when data already present should be a noop.
zdc.push()
zkclient.create.assert_not_called()
treadmill.zkutils.ensure_deleted.assert_not_called()
treadmill.zkutils.ensure_deleted.reset_mock()
zkclient.create.reset_mock()
def test_rm_data(self):
"""Test deletion of cache data entry.
"""
# pylint: disable=protected-access
zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir)
# Deleting non-existent items.
zdc._cached = {}
zdc.rm_data('not there')
treadmill.fs.rm_safe.assert_not_called()
treadmill.fs.rm_safe.reset_mock()
# Asking for data.
zdc._cached = {
'TEST': [
zkdatacache.ZkCachedEntry(fname='a#chk2',
chksum='chk2',
ctime='0'),
zkdatacache.ZkCachedEntry(fname='a#chk1',
chksum='chk1',
ctime='0'),
],
}
zdc.rm_data('TEST')
self.assertEqual(zdc._cached, {})
treadmill.fs.rm_safe.assert_has_calls([
mock.call('a#chk1'),
mock.call('a#chk2'),
],
any_order=True)
treadmill.fs.rm_safe.reset_mock()
def test__trim_cache(self):
"""Test _trim operation leaves the right number of entries in the
cache.
"""
# pylint: disable=protected-access
zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir)
zdc._cached = {}
self.assertRaises(KeyError, zdc._trim_cache, 'does not exists')
self.assertEqual(zdc._cached, {})
treadmill.fs.rm_safe.assert_not_called()
treadmill.fs.rm_safe.reset_mock()
zdc._cached = {
'TEST': [
zkdatacache.ZkCachedEntry(fname='a#chk2',
chksum='chk2',
ctime='0'),
zkdatacache.ZkCachedEntry(fname='a#chk1',
chksum='chk1',
ctime='0'),
],
}
zdc._trim_cache('TEST')
self.assertEqual(
zdc._cached, {
'TEST': [
zkdatacache.ZkCachedEntry(
fname='a#chk2', chksum='chk2', ctime='0'),
]
})
treadmill.fs.rm_safe.assert_called_with('a#chk1')
treadmill.fs.rm_safe.reset_mock()
def test_push(self):
"""Test pushing data from the cache to Zookeeper.
"""
# pylint: disable=protected-access
zk_content = {
'zk': {
'path': {
'FOO#chk0#0000000039': b'notbar',
'FOO#chk1#0000000042': b'bar',
'BAR#chk_some#0000000040': b'notfoo',
'BAR#chk_other#0000000041': b'foo',
}
}
}
self.make_mock_zk(zk_content)
zkclient = kazoo.client.KazooClient()
zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir)
zdc._cached = {
'FOO': [
zkdatacache.ZkCachedEntry(fname='a#chk2',
chksum='chk2',
ctime='0'),
zkdatacache.ZkCachedEntry(fname='a#chk1',
chksum='chk1',
ctime='0'),
]
}
# Test "normal" pushing.
zdc.push()
zkclient.create.assert_called_with('/zk/path/FOO#chk2' + '#',
b'foo',
acl=mock.ANY,
makepath=True,
sequence=True)
treadmill.zkutils.ensure_deleted.assert_called_once_with(
zkclient, '/zk/path/FOO#chk0#0000000039')
treadmill.zkutils.ensure_deleted.reset_mock()
zkclient.create.reset_mock()
# Test expunge pushing.
zk_content['zk']['path']['FOO#chk2#0000000043'] = b'foo'
zdc.refresh_zk(zkclient.get_children('/zk/path'))
zdc.push(expunge=True)
treadmill.zkutils.ensure_deleted.assert_has_calls([
mock.call(zkclient, '/zk/path/FOO#chk0#0000000039'),
mock.call(zkclient, '/zk/path/BAR#chk_some#0000000040'),
mock.call(zkclient, '/zk/path/BAR#chk_other#0000000041'),
mock.call(zkclient, '/zk/path/FOO#chk1#0000000042'),
],
any_order=True)
self.assertEqual(treadmill.zkutils.ensure_deleted.call_count, 4)
def test_get_data(self):
"""Test cache data retrieaval.
"""
# pylint: disable=protected-access
zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir)
# Asking for non-existent items.
zdc._cached = {}
self.assertRaises(KeyError, zdc.get_data, 'not there')
# Asking for data.
zdc._cached = {
'TEST': [
zkdatacache.ZkCachedEntry(fname='a#chk2',
chksum='chk2',
ctime='0'),
]
}
res = zdc.get_data('TEST')
io.open.assert_called_with('a#chk2', mode='rb')
mock_file = io.open.return_value
self.assertEqual(res, mock_file)
self.assertEqual(mock_file.checksum, 'chk2')
def test_push(self):
"""Test pushing data from the cache to Zookeeper.
"""
# pylint: disable=protected-access
zkclient = kazoo.client.KazooClient()
zkclient.get_children.side_effect = [
[
'FOO#chk0#0000000039',
'FOO#chk1#0000000042',
'BAR#chk_some#0000000040',
'BAR#chk_other#0000000041',
],
[
'FOO#chk0#0000000039',
'FOO#chk1#0000000042',
'FOO#chk2#0000000043',
'BAR#chk_some#0000000040',
'BAR#chk_other#0000000041',
],
]
zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir)
zdc._cached = {
'FOO': [
zkdatacache.ZkCachedEntry(
fname='a#chk2', chksum='chk2', ctime='0'
),
zkdatacache.ZkCachedEntry(
fname='a#chk1', chksum='chk1', ctime='0'
),
]
}
# Test "normal" pushing.
zdc.push()
zkclient.create.assert_called_with(
'/zk/path/FOO#chk2' + '#',
b'foo',
makepath=True, sequence=True
)
self.assertEqual(
treadmill.zkutils.ensure_deleted.call_args_list,
[
mock.call(zkclient, '/zk/path/FOO#chk1#0000000042'),
mock.call(zkclient, '/zk/path/FOO#chk0#0000000039'),
]
)
treadmill.zkutils.ensure_deleted.reset_mock()
zkclient.get_children.side_effect = None
zkclient.create.reset_mock()
# Test expunge pushing.
zkclient.get_children.return_value = [
'FOO#chk0#0000000039',
'FOO#chk1#0000000042',
'FOO#chk2#0000000043',
'BAR#chk_some#0000000040',
'BAR#chk_other#0000000041',
]
zdc.refresh_zk(zkclient.get_children('/zk/path'))
zdc.push(expunge=True)
treadmill.zkutils.ensure_deleted.assert_has_calls(
[
mock.call(zkclient, '/zk/path/FOO#chk0#0000000039'),
mock.call(zkclient, '/zk/path/BAR#chk_some#0000000040'),
mock.call(zkclient, '/zk/path/BAR#chk_other#0000000041'),
mock.call(zkclient, '/zk/path/FOO#chk1#0000000042'),
],
any_order=True
)
self.assertEqual(
treadmill.zkutils.ensure_deleted.call_count,
4
)
def test_pull(self):
"""Test pulling data from Zookeeper into the cache.
"""
# pylint: disable=protected-access
zk_content = {
'zk': {
'path': {
'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33#00000042': (
b'foo'
)
}
}
}
self.make_mock_zk(zk_content)
zkclient = treadmill.zkutils.ZkClient()
test_cache_entry = zkdatacache.ZkCachedEntry(
fname='TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33',
chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33',
ctime='0'
)
zkdatacache.ZkDataCache._add_data_bytes.return_value = test_cache_entry
zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir)
zdc._cached = {
'FOO': [
zkdatacache.ZkCachedEntry(
fname='a#chk2', chksum='chk2', ctime='0'
),
zkdatacache.ZkCachedEntry(
fname='a#chk1', chksum='chk1', ctime='0'
),
]
}
# Test "normal" pulling.
zdc.pull()
zkclient.get.assert_called_with(
(
'/zk/path/TEST#'
'0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33#00000042'
)
)
zdc._add_data_bytes.assert_called_with(
'TEST',
b'foo',
chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
)
zdc._trim_cache.assert_called_once_with('TEST')
self.assertEqual(
zdc._cached,
{
'TEST': [
test_cache_entry
],
'FOO': [
zkdatacache.ZkCachedEntry(
fname='a#chk2', chksum='chk2', ctime='0'
),
zkdatacache.ZkCachedEntry(
fname='a#chk1', chksum='chk1', ctime='0'
),
],
}
)
zdc.rm_data.assert_not_called()
zkclient.get.reset_mock()
zkclient.get_children.reset_mock()
zkclient.exists.reset_mock()
zdc._add_data_bytes.reset_mock()
zdc._trim_cache.reset_mock()
zdc.rm_data.reset_mock()
# Second pull should result in noop
zdc.pull()
zdc._add_data_bytes.assert_not_called()
zdc.rm_data.assert_not_called()
zdc._trim_cache.assert_called_once_with('TEST')
self.assertEqual(
zdc._cached,
{
'TEST': [
test_cache_entry
],
'FOO': [
zkdatacache.ZkCachedEntry(
fname='a#chk2', chksum='chk2', ctime='0'
),
zkdatacache.ZkCachedEntry(
fname='a#chk1', chksum='chk1', ctime='0'
),
],
}
)
zkclient.get.reset_mock()
zkclient.get_children.reset_mock()
zkclient.exists.reset_mock()
zdc._add_data_bytes.reset_mock()
zdc._trim_cache.reset_mock()
zdc.rm_data.reset_mock()
# Pull with expunge should remove all extra entries.
zdc.pull(expunge=True)
zdc._add_data_bytes.assert_not_called()
zdc.rm_data.assert_called_with('FOO')
zdc._trim_cache.assert_called_with('TEST')
def test_refresh_cache(self):
"""Test refresh of local cache data.
"""
os.listdir.return_value = []
zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir)
zdc.refresh_cache()
self.assertEqual(
zdc.cached,
{}
)
os.listdir.return_value = [
'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33',
'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33',
'FOO#some_check',
'BAR#other_check',
'bad_file',
]
os.stat.side_effect = [
os.stat_result((42,) * 10),
os.stat_result((43,) * 10),
os.stat_result((44,) * 10),
os.stat_result((45,) * 10),
]
zdc.refresh_cache()
os.stat.assert_has_calls(
[
mock.call(
os.path.join(
self.cachedir,
'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
)
),
mock.call(
os.path.join(
self.cachedir,
'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33',
)
),
mock.call(
os.path.join(
self.cachedir,
'FOO#some_check',
)
),
mock.call(
os.path.join(
self.cachedir,
'BAR#other_check',
)
),
]
)
self.assertEqual(
zdc.cached,
{
'TEST': [
zkdatacache.ZkCachedEntry(
fname=os.path.join(
self.cachedir,
'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
),
chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33',
ctime=43
),
zkdatacache.ZkCachedEntry(
fname=os.path.join(
self.cachedir,
'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
),
chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33',
ctime=42
),
],
'BAR': [
zkdatacache.ZkCachedEntry(
fname=os.path.join(
self.cachedir,
'BAR#other_check'
),
chksum='other_check',
ctime=45
),
],
'FOO': [
zkdatacache.ZkCachedEntry(
fname=os.path.join(
self.cachedir,
'FOO#some_check'
),
chksum='some_check',
ctime=44
),
],
}
)
