def test_simple_operations_with_raw_keys_and_values(
minion_opts, profile_name, prefix, etcd_version
):
if etcd_version in (EtcdVersion.v2, EtcdVersion.v3_v2_mode):
pytest.skip("Not testing with raw keys using v2")
modified_opts = copy.deepcopy(minion_opts)
modified_opts[profile_name]["etcd.raw_keys"] = True
modified_opts[profile_name]["etcd.raw_values"] = True
etcd_client = get_conn(modified_opts, profile=profile_name)
assert not etcd_client.get("{}/mtg/ambush".format(prefix))
assert etcd_client.set("{}/mtg/ambush".format(prefix), "viper") == b"viper"
assert etcd_client.get("{}/mtg/ambush".format(prefix)) == b"viper"
assert etcd_client.set("{}/mtg/counter".format(prefix), "spell") == b"spell"
assert etcd_client.tree("{}/mtg".format(prefix)) == {
b"ambush": b"viper",
b"counter": b"spell",
}
assert etcd_client.ls("{}/mtg".format(prefix)) == {
"{}/mtg".format(prefix).encode("UTF-8"): {
"{}/mtg/ambush".format(prefix).encode("UTF-8"): b"viper",
"{}/mtg/counter".format(prefix).encode("UTF-8"): b"spell",
},
}
assert etcd_client.delete("{}/mtg/ambush".format(prefix))
assert etcd_client.delete("{}/mtg".format(prefix), recurse=True)
assert not etcd_client.get("{}/mtg".format(prefix), recurse=True)
def ext_pillar(minion_id, pillar, conf): # pylint: disable=W0613
'''
Check etcd for all data
'''
comps = conf.split()
profile = None
if comps[0]:
profile = comps[0]
client = etcd_util.get_conn(__opts__, profile)
path = '/'
if len(comps) > 1 and comps[1].startswith('root='):
path = comps[1].replace('root=', '')
# put the minion's ID in the path if necessary
path %= {'minion_id': minion_id}
try:
pillar = etcd_util.tree(client, path)
except KeyError:
log.error('No such key in etcd profile {0}: {1}'.format(profile, path))
pillar = {}
return pillar
def test_expand(use_v2, client_name):
if use_v2:
pytest.skip("No expand in etcd v2 adapter")
with patch(client_name, autospec=True) as mock:
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
some_data = {
"/test/x/y/a": "1",
"/test/x/y/b": "2",
"/test/m/j": "3",
"/test/z": "4",
}
result = {
"test": {
"x": {"y": {"a": "1", "b": "2"}},
"m": {"j": "3"},
"z": "4",
},
}
assert client._expand(some_data) == result
def test_tree(use_v2, client_name):
"""
Test recursive gets
"""
with patch(client_name, autospec=True):
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
if use_v2:
with patch.object(client, "read", autospec=True) as mock:
c1, c2 = MagicMock(), MagicMock()
c1.__iter__.return_value = [
MagicMock(key="/x/a", value="1"),
MagicMock(key="/x/b", value="2"),
MagicMock(key="/x/c", dir=True),
]
c2.__iter__.return_value = [MagicMock(key="/x/c/d", value="3")]
mock.side_effect = iter(
[MagicMock(children=c1), MagicMock(children=c2)]
)
assert client.tree("/x") == {"a": "1", "b": "2", "c": {"d": "3"}}
mock.assert_any_call("/x")
mock.assert_any_call("/x/c")
# iter(list(Exception)) works correctly with both mock<1.1 and mock>=1.1
mock.side_effect = iter([etcd.EtcdKeyNotFound()])
assert client.tree("not-found") is None
mock.side_effect = ValueError
assert client.tree("/x") is None
mock.side_effect = Exception
with pytest.raises(Exception):
client.tree("some-error")
else:
with patch.object(client, "read", autospec=True) as mock:
mock.return_value = [
MagicMock(key="/x/a", value="1"),
MagicMock(key="/x/b", value="2"),
MagicMock(key="/x/c/d", value="3"),
]
assert client.tree("/x") == {"a": "1", "b": "2", "c": {"d": "3"}}
mock.assert_called_with("/x", recurse=True)
def test_update(use_v2, client_name):
with patch(client_name, autospec=True) as mock:
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
some_data = {
"/x/y/a": "1",
"x": {"y": {"b": "3"}},
"m/j/": "3",
"z": "4",
"d": {},
}
result = {
"/test/x/y/a": "1",
"/test/x/y/b": "2",
"/test/m/j": "3",
"/test/z": "4",
"/test/d": True,
}
flatten_result = {
"/test/x/y/a": "1",
"/test/x/y/b": "2",
"/test/m/j": "3",
"/test/z": "4",
"/test/d": {},
}
client._flatten = MagicMock(return_value=flatten_result)
assert client.update("/some/key", path="/blah") is None
with patch.object(client, "write", autospec=True) as write_mock:
def write_return(key, val, ttl=None, directory=None):
return result.get(key, None)
write_mock.side_effect = write_return
if not use_v2:
result.pop("/test/d")
assert client.update(some_data, path="/test") == result
client._flatten.assert_called_with(some_data, "/test")
assert write_mock.call_count == 5 if use_v2 else 4
def test_rm(use_v2, client_name):
with patch(client_name, autospec=True) as mock:
etcd_client = mock.return_value
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
if use_v2:
etcd_client.delete.return_value = True
assert client.rm("/some-key")
etcd_client.delete.assert_called_with("/some-key", recursive=False)
assert client.rm("/some-dir", recurse=True)
etcd_client.delete.assert_called_with("/some-dir", recursive=True)
etcd_client.delete.side_effect = etcd.EtcdNotFile()
assert client.rm("/some-dir") is None
etcd_client.delete.side_effect = etcd.EtcdDirNotEmpty()
assert client.rm("/some-key") is None
etcd_client.delete.side_effect = etcd.EtcdRootReadOnly()
assert client.rm("/") is None
etcd_client.delete.side_effect = ValueError
assert client.rm("/some-dir") is None
etcd_client.delete.side_effect = Exception
with pytest.raises(Exception):
client.rm("some-dir")
else:
etcd_client.delete_range.return_value = MagicMock(deleted=1)
assert client.rm("/some-key")
etcd_client.delete_range.assert_called_with("/some-key", prefix=False)
etcd_client.delete_range.return_value = MagicMock(deleted=0)
assert client.rm("/some-key", recurse=True) is None
etcd_client.delete_range.assert_called_with("/some-key", prefix=True)
delattr(etcd_client.delete_range.return_value, "deleted")
assert not client.rm("/some-key")
etcd_client.delete_range.assert_called_with("/some-key", prefix=False)
def test_flatten(use_v2, client_name):
with patch(client_name, autospec=True) as mock:
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
some_data = {
"/x/y/a": "1",
"x": {"y": {"b": "2"}},
"m/j/": "3",
"z": "4",
"d": {},
}
result_path = {
"/test/x/y/a": "1",
"/test/x/y/b": "2",
"/test/m/j": "3",
"/test/z": "4",
"/test/d": {},
}
result_nopath = {
"/x/y/a": "1",
"/x/y/b": "2",
"/m/j": "3",
"/z": "4",
"/d": {},
}
result_root = {
"/x/y/a": "1",
"/x/y/b": "2",
"/m/j": "3",
"/z": "4",
"/d": {},
}
assert client._flatten(some_data, path="/test") == result_path
assert client._flatten(some_data, path="/") == result_root
assert client._flatten(some_data) == result_nopath
def test_get(client_name, use_v2):
"""
Test if it get a value from etcd, by direct path
"""
with patch(client_name, autospec=True):
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
if use_v2:
with patch.object(client, "read", autospec=True) as mock:
mock.return_value = MagicMock(value="stack")
assert client.get("salt") == "stack"
mock.assert_called_with("salt")
# iter(list(Exception)) works correctly with both mock<1.1 and mock>=1.1
mock.side_effect = iter([etcd.EtcdKeyNotFound()])
assert client.get("not-found") is None
mock.side_effect = iter([etcd.EtcdConnectionFailed()])
assert client.get("watching") is None
# python 2.6 test
mock.side_effect = ValueError
assert client.get("not-found") is None
mock.side_effect = Exception
with pytest.raises(Exception):
client.get("some-error")
else:
with patch.object(client, "read", autospec=True) as mock:
mock.return_value = [MagicMock(value="stack")]
assert client.get("salt") == "stack"
mock.assert_called_with("salt")
# Get with recurse now delegates to client.tree
with patch.object(client, "tree", autospec=True) as tree_mock:
tree_mock.return_value = {"salt": "stack"}
assert client.get("salt", recurse=True) == {"salt": "stack"}
tree_mock.assert_called_with("salt")
def etcd_client(minion_opts, profile_name):
return get_conn(minion_opts, profile=profile_name)
def test_etcd_client_creation_with_get_conn(minion_opts, profile_name):
"""
Client creation using get_conn, just need to assert no errors.
"""
get_conn(minion_opts, profile=profile_name)
def test_watch(use_v2, client_name):
with patch(client_name, autospec=True):
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
if use_v2:
with patch.object(client, "read", autospec=True) as mock:
mock.return_value = MagicMock(
value="stack", key="/some-key", modifiedIndex=1, dir=False
)
assert client.watch("/some-key") == {
"value": "stack",
"key": "/some-key",
"mIndex": 1,
"changed": True,
"dir": False,
}
mock.assert_called_with(
"/some-key",
wait=True,
recurse=False,
timeout=0,
start_revision=None,
)
mock.side_effect = iter(
[etcd_util.EtcdUtilWatchTimeout, mock.return_value]
)
assert client.watch("/some-key") == {
"value": "stack",
"changed": False,
"mIndex": 1,
"key": "/some-key",
"dir": False,
}
mock.side_effect = iter(
[etcd_util.EtcdUtilWatchTimeout, etcd.EtcdKeyNotFound]
)
assert client.watch("/some-key") == {
"value": None,
"changed": False,
"mIndex": 0,
"key": "/some-key",
"dir": False,
}
mock.side_effect = iter([etcd_util.EtcdUtilWatchTimeout, ValueError])
assert client.watch("/some-key") == {}
mock.side_effect = None
mock.return_value = MagicMock(
value="stack", key="/some-key", modifiedIndex=1, dir=True
)
assert client.watch(
"/some-dir", recurse=True, timeout=5, start_revision=10
) == {
"value": "stack",
"key": "/some-key",
"mIndex": 1,
"changed": True,
"dir": True,
}
mock.assert_called_with(
"/some-dir", wait=True, recurse=True, timeout=5, start_revision=10
)
# iter(list(Exception)) works correctly with both mock<1.1 and mock>=1.1
mock.side_effect = iter([MaxRetryError(None, None)])
assert client.watch("/some-key") == {}
mock.side_effect = iter([etcd.EtcdConnectionFailed()])
assert client.watch("/some-key") is None
mock.side_effect = None
mock.return_value = None
assert client.watch("/some-key") == {}
else:
with patch.object(client, "read", autospec=True) as mock:
mock.return_value = MagicMock(
value="stack", key="/some-key", mod_revision=1
)
assert client.watch("/some-key") == {
"value": "stack",
"key": "/some-key",
"mIndex": 1,
"changed": True,
"dir": False,
}
mock.assert_called_with(
"/some-key",
wait=True,
recurse=False,
timeout=0,
start_revision=None,
)
mock.return_value = MagicMock(
value="stack", key="/some-key", mod_revision=1
)
assert client.watch(
"/some-key", recurse=True, timeout=5, start_revision=10
) == {
"value": "stack",
"key": "/some-key",
"mIndex": 1,
"changed": True,
"dir": False,
}
mock.assert_called_with(
"/some-key", wait=True, recurse=True, timeout=5, start_revision=10
)
mock.side_effect = None
mock.return_value = None
assert client.watch("/some-key") is None
def test_read(client_name, use_v2):
"""
Test to make sure we interact with etcd correctly
"""
with patch(client_name, autospec=True) as mock:
etcd_client = mock.return_value
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
if use_v2:
etcd_return = MagicMock(value="salt")
etcd_client.read.return_value = etcd_return
assert client.read("/salt") == etcd_return
etcd_client.read.assert_called_with(
"/salt", recursive=False, wait=False, timeout=None
)
client.read("salt", True, True, 10, 5)
etcd_client.read.assert_called_with(
"salt", recursive=True, wait=True, timeout=10, waitIndex=5
)
etcd_client.read.side_effect = etcd.EtcdKeyNotFound
with pytest.raises(etcd.EtcdKeyNotFound):
client.read("salt")
etcd_client.read.side_effect = etcd.EtcdConnectionFailed
with pytest.raises(etcd.EtcdConnectionFailed):
client.read("salt")
etcd_client.read.side_effect = etcd.EtcdValueError
with pytest.raises(etcd.EtcdValueError):
client.read("salt")
etcd_client.read.side_effect = ValueError
with pytest.raises(ValueError):
client.read("salt")
etcd_client.read.side_effect = ReadTimeoutError(None, None, None)
with pytest.raises(etcd.EtcdConnectionFailed):
client.read("salt")
etcd_client.read.side_effect = MaxRetryError(None, None)
with pytest.raises(etcd.EtcdConnectionFailed):
client.read("salt")
else:
etcd_return = MagicMock(kvs=[MagicMock(value="salt")])
etcd_client.range.return_value = etcd_return
assert client.read("/salt") == etcd_return.kvs
etcd_client.range.assert_called_with("/salt", prefix=False)
etcd_client.range.side_effect = Exception
assert client.read("/salt") is None
watcher_mock = MagicMock()
with patch.object(etcd_client, "Watcher", return_value=watcher_mock):
client.read("salt", True, True, 10, 5)
etcd_client.range.assert_called_with("/salt", prefix=False)
watcher_mock.watch_once.assert_called_with(timeout=10)
watcher_mock.watch_once.side_effect = Exception
assert client.read("salt", True, True, 10, 5) is None
def test_write(use_v2, client_name):
with patch(client_name, autospec=True) as mock:
etcd_client = mock.return_value
client = etcd_util.get_conn(
{"etcd.require_v2": use_v2, "etcd.encode_values": False}
)
if use_v2:
etcd_client.write.return_value = MagicMock(value="salt")
assert client.write("/some-key", "salt") == "salt"
etcd_client.write.assert_called_with(
"/some-key", "salt", ttl=None, dir=False
)
assert client.write("/some-key", "salt", ttl=5) == "salt"
etcd_client.write.assert_called_with("/some-key", "salt", ttl=5, dir=False)
etcd_client.write.return_value = MagicMock(dir=True)
assert client.write("/some-dir", "salt", ttl=0, directory=True)
etcd_client.write.assert_called_with("/some-dir", None, ttl=0, dir=True)
# Check when a file is attempted to be written to a read-only root
etcd_client.write.side_effect = etcd.EtcdRootReadOnly()
assert client.write("/", "some-val", directory=False) is None
# Check when a directory is attempted to be written to a read-only root
etcd_client.write.side_effect = etcd.EtcdRootReadOnly()
assert client.write("/", None, directory=True) is None
# Check when a file is attempted to be written when unable to connect to the service
etcd_client.write.side_effect = MaxRetryError(None, None)
assert client.write("/some-key", "some-val", directory=False) is None
# Check when a directory is attempted to be written when unable to connect to the service
etcd_client.write.side_effect = MaxRetryError(None, None)
assert client.write("/some-dir", None, directory=True) is None
# Check when a file is attempted to be written to a directory that already exists (name-collision)
etcd_client.write.side_effect = etcd.EtcdNotFile()
assert client.write("/some-dir", "some-val", directory=False) is None
# Check when a directory is attempted to be written to a file that already exists (name-collision)
etcd_client.write.side_effect = etcd.EtcdNotDir()
assert client.write("/some-key", None, directory=True) is None
# Check when a directory is attempted to be written to a directory that already exists (update-ttl)
etcd_client.write.side_effect = etcd.EtcdNotFile()
assert client.write("/some-dir", None, directory=True)
etcd_client.write.side_effect = ValueError
assert client.write("/some-key", "some-val") is None
etcd_client.write.side_effect = Exception
with pytest.raises(Exception):
client.set("some-key", "some-val")
else:
with pytest.raises(etcd_util.Etcd3DirectoryException):
client.write("key", None, directory=True)
with patch.object(client, "get", autospec=True) as get_mock:
get_mock.return_value = "stack"
assert client.write("salt", "stack") == "stack"
etcd_client.put.assert_called_with("salt", "stack")
lease_mock = MagicMock(ID=1)
with patch.object(etcd_client, "Lease", return_value=lease_mock):
assert client.write("salt", "stack", ttl=5) == "stack"
etcd_client.put.assert_called_with("salt", "stack", lease=1)
