def test_push_noop(self): """Test pushing data from the cache to Zookeeper. """ # pylint: disable=protected-access zk_content = { 'zk': { 'path': { 'FOO#chk2#0000000042': b'foo', } } } self.make_mock_zk(zk_content) zkclient = treadmill.zkutils.ZkClient() zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir) zdc._cached = { 'FOO': [ zkdatacache.ZkCachedEntry( fname='a#chk2', chksum='chk2', ctime='0' ), zkdatacache.ZkCachedEntry( fname='a#chk1', chksum='chk1', ctime='0' ), ] } # Pushing when data already present should be a noop. zdc.push() zkclient.create.assert_not_called() treadmill.zkutils.ensure_deleted.assert_not_called() treadmill.zkutils.ensure_deleted.reset_mock() zkclient.create.reset_mock()
def test_rm_data(self): """Test deletion of cache data entry. """ # pylint: disable=protected-access zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir) # Deleting non-existent items. zdc._cached = {} zdc.rm_data('not there') treadmill.fs.rm_safe.assert_not_called() treadmill.fs.rm_safe.reset_mock() # Asking for data. zdc._cached = { 'TEST': [ zkdatacache.ZkCachedEntry(fname='a#chk2', chksum='chk2', ctime='0'), zkdatacache.ZkCachedEntry(fname='a#chk1', chksum='chk1', ctime='0'), ], } zdc.rm_data('TEST') self.assertEqual(zdc._cached, {}) treadmill.fs.rm_safe.assert_has_calls([ mock.call('a#chk1'), mock.call('a#chk2'), ], any_order=True) treadmill.fs.rm_safe.reset_mock()
def test__trim_cache(self): """Test _trim operation leaves the right number of entries in the cache. """ # pylint: disable=protected-access zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir) zdc._cached = {} self.assertRaises(KeyError, zdc._trim_cache, 'does not exists') self.assertEqual(zdc._cached, {}) treadmill.fs.rm_safe.assert_not_called() treadmill.fs.rm_safe.reset_mock() zdc._cached = { 'TEST': [ zkdatacache.ZkCachedEntry(fname='a#chk2', chksum='chk2', ctime='0'), zkdatacache.ZkCachedEntry(fname='a#chk1', chksum='chk1', ctime='0'), ], } zdc._trim_cache('TEST') self.assertEqual( zdc._cached, { 'TEST': [ zkdatacache.ZkCachedEntry( fname='a#chk2', chksum='chk2', ctime='0'), ] }) treadmill.fs.rm_safe.assert_called_with('a#chk1') treadmill.fs.rm_safe.reset_mock()
def test_push(self): """Test pushing data from the cache to Zookeeper. """ # pylint: disable=protected-access zk_content = { 'zk': { 'path': { 'FOO#chk0#0000000039': b'notbar', 'FOO#chk1#0000000042': b'bar', 'BAR#chk_some#0000000040': b'notfoo', 'BAR#chk_other#0000000041': b'foo', } } } self.make_mock_zk(zk_content) zkclient = kazoo.client.KazooClient() zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir) zdc._cached = { 'FOO': [ zkdatacache.ZkCachedEntry(fname='a#chk2', chksum='chk2', ctime='0'), zkdatacache.ZkCachedEntry(fname='a#chk1', chksum='chk1', ctime='0'), ] } # Test "normal" pushing. zdc.push() zkclient.create.assert_called_with('/zk/path/FOO#chk2' + '#', b'foo', acl=mock.ANY, makepath=True, sequence=True) treadmill.zkutils.ensure_deleted.assert_called_once_with( zkclient, '/zk/path/FOO#chk0#0000000039') treadmill.zkutils.ensure_deleted.reset_mock() zkclient.create.reset_mock() # Test expunge pushing. zk_content['zk']['path']['FOO#chk2#0000000043'] = b'foo' zdc.refresh_zk(zkclient.get_children('/zk/path')) zdc.push(expunge=True) treadmill.zkutils.ensure_deleted.assert_has_calls([ mock.call(zkclient, '/zk/path/FOO#chk0#0000000039'), mock.call(zkclient, '/zk/path/BAR#chk_some#0000000040'), mock.call(zkclient, '/zk/path/BAR#chk_other#0000000041'), mock.call(zkclient, '/zk/path/FOO#chk1#0000000042'), ], any_order=True) self.assertEqual(treadmill.zkutils.ensure_deleted.call_count, 4)
def test_get_data(self): """Test cache data retrieaval. """ # pylint: disable=protected-access zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir) # Asking for non-existent items. zdc._cached = {} self.assertRaises(KeyError, zdc.get_data, 'not there') # Asking for data. zdc._cached = { 'TEST': [ zkdatacache.ZkCachedEntry(fname='a#chk2', chksum='chk2', ctime='0'), ] } res = zdc.get_data('TEST') io.open.assert_called_with('a#chk2', mode='rb') mock_file = io.open.return_value self.assertEqual(res, mock_file) self.assertEqual(mock_file.checksum, 'chk2')
def test_push(self): """Test pushing data from the cache to Zookeeper. """ # pylint: disable=protected-access zkclient = kazoo.client.KazooClient() zkclient.get_children.side_effect = [ [ 'FOO#chk0#0000000039', 'FOO#chk1#0000000042', 'BAR#chk_some#0000000040', 'BAR#chk_other#0000000041', ], [ 'FOO#chk0#0000000039', 'FOO#chk1#0000000042', 'FOO#chk2#0000000043', 'BAR#chk_some#0000000040', 'BAR#chk_other#0000000041', ], ] zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir) zdc._cached = { 'FOO': [ zkdatacache.ZkCachedEntry( fname='a#chk2', chksum='chk2', ctime='0' ), zkdatacache.ZkCachedEntry( fname='a#chk1', chksum='chk1', ctime='0' ), ] } # Test "normal" pushing. zdc.push() zkclient.create.assert_called_with( '/zk/path/FOO#chk2' + '#', b'foo', makepath=True, sequence=True ) self.assertEqual( treadmill.zkutils.ensure_deleted.call_args_list, [ mock.call(zkclient, '/zk/path/FOO#chk1#0000000042'), mock.call(zkclient, '/zk/path/FOO#chk0#0000000039'), ] ) treadmill.zkutils.ensure_deleted.reset_mock() zkclient.get_children.side_effect = None zkclient.create.reset_mock() # Test expunge pushing. zkclient.get_children.return_value = [ 'FOO#chk0#0000000039', 'FOO#chk1#0000000042', 'FOO#chk2#0000000043', 'BAR#chk_some#0000000040', 'BAR#chk_other#0000000041', ] zdc.refresh_zk(zkclient.get_children('/zk/path')) zdc.push(expunge=True) treadmill.zkutils.ensure_deleted.assert_has_calls( [ mock.call(zkclient, '/zk/path/FOO#chk0#0000000039'), mock.call(zkclient, '/zk/path/BAR#chk_some#0000000040'), mock.call(zkclient, '/zk/path/BAR#chk_other#0000000041'), mock.call(zkclient, '/zk/path/FOO#chk1#0000000042'), ], any_order=True ) self.assertEqual( treadmill.zkutils.ensure_deleted.call_count, 4 )
def test_pull(self): """Test pulling data from Zookeeper into the cache. """ # pylint: disable=protected-access zk_content = { 'zk': { 'path': { 'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33#00000042': ( b'foo' ) } } } self.make_mock_zk(zk_content) zkclient = treadmill.zkutils.ZkClient() test_cache_entry = zkdatacache.ZkCachedEntry( fname='TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33', chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33', ctime='0' ) zkdatacache.ZkDataCache._add_data_bytes.return_value = test_cache_entry zdc = zkdatacache.ZkDataCache(zkclient, '/zk/path', self.cachedir) zdc._cached = { 'FOO': [ zkdatacache.ZkCachedEntry( fname='a#chk2', chksum='chk2', ctime='0' ), zkdatacache.ZkCachedEntry( fname='a#chk1', chksum='chk1', ctime='0' ), ] } # Test "normal" pulling. zdc.pull() zkclient.get.assert_called_with( ( '/zk/path/TEST#' '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33#00000042' ) ) zdc._add_data_bytes.assert_called_with( 'TEST', b'foo', chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33' ) zdc._trim_cache.assert_called_once_with('TEST') self.assertEqual( zdc._cached, { 'TEST': [ test_cache_entry ], 'FOO': [ zkdatacache.ZkCachedEntry( fname='a#chk2', chksum='chk2', ctime='0' ), zkdatacache.ZkCachedEntry( fname='a#chk1', chksum='chk1', ctime='0' ), ], } ) zdc.rm_data.assert_not_called() zkclient.get.reset_mock() zkclient.get_children.reset_mock() zkclient.exists.reset_mock() zdc._add_data_bytes.reset_mock() zdc._trim_cache.reset_mock() zdc.rm_data.reset_mock() # Second pull should result in noop zdc.pull() zdc._add_data_bytes.assert_not_called() zdc.rm_data.assert_not_called() zdc._trim_cache.assert_called_once_with('TEST') self.assertEqual( zdc._cached, { 'TEST': [ test_cache_entry ], 'FOO': [ zkdatacache.ZkCachedEntry( fname='a#chk2', chksum='chk2', ctime='0' ), zkdatacache.ZkCachedEntry( fname='a#chk1', chksum='chk1', ctime='0' ), ], } ) zkclient.get.reset_mock() zkclient.get_children.reset_mock() zkclient.exists.reset_mock() zdc._add_data_bytes.reset_mock() zdc._trim_cache.reset_mock() zdc.rm_data.reset_mock() # Pull with expunge should remove all extra entries. zdc.pull(expunge=True) zdc._add_data_bytes.assert_not_called() zdc.rm_data.assert_called_with('FOO') zdc._trim_cache.assert_called_with('TEST')
def test_refresh_cache(self): """Test refresh of local cache data. """ os.listdir.return_value = [] zdc = zkdatacache.ZkDataCache(None, '/zk/path', self.cachedir) zdc.refresh_cache() self.assertEqual( zdc.cached, {} ) os.listdir.return_value = [ 'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33', 'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33', 'FOO#some_check', 'BAR#other_check', 'bad_file', ] os.stat.side_effect = [ os.stat_result((42,) * 10), os.stat_result((43,) * 10), os.stat_result((44,) * 10), os.stat_result((45,) * 10), ] zdc.refresh_cache() os.stat.assert_has_calls( [ mock.call( os.path.join( self.cachedir, 'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33' ) ), mock.call( os.path.join( self.cachedir, 'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33', ) ), mock.call( os.path.join( self.cachedir, 'FOO#some_check', ) ), mock.call( os.path.join( self.cachedir, 'BAR#other_check', ) ), ] ) self.assertEqual( zdc.cached, { 'TEST': [ zkdatacache.ZkCachedEntry( fname=os.path.join( self.cachedir, 'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33' ), chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33', ctime=43 ), zkdatacache.ZkCachedEntry( fname=os.path.join( self.cachedir, 'TEST#0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33' ), chksum='0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33', ctime=42 ), ], 'BAR': [ zkdatacache.ZkCachedEntry( fname=os.path.join( self.cachedir, 'BAR#other_check' ), chksum='other_check', ctime=45 ), ], 'FOO': [ zkdatacache.ZkCachedEntry( fname=os.path.join( self.cachedir, 'FOO#some_check' ), chksum='some_check', ctime=44 ), ], } )