def wait_command(command, result, count=1, ns=namespace, retries=max_retries):
with Then(f"{command} should return {result}"):
for i in range(1, retries):
res = launch(command, ok_to_fail=True, ns=ns)
if res == result:
break
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert res == result, error()
def kube_wait_objects(chi, ns, objects):
with Then(f"{objects[0]} statefulsets, {objects[1]} pods and {objects[2]} services should be created"):
for i in range(1,max_retries):
counts = kube_count_resources(ns, f"-l clickhouse.altinity.com/chi={chi}")
if counts == objects:
break
with Then("Not ready. Wait for " + str(i*5) + " seconds"):
time.sleep(i*5)
assert counts == objects, error()
def delete_user_from_ldap(user, node=None, exitcode=0):
"""Delete user entry from LDAP."""
if node is None:
node = current().context.ldap_node
r = node.command(
f"ldapdelete -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin \"{user['dn']}\""
)
if exitcode is not None:
assert r.exitcode == exitcode, error()
def verify_ldap_user_exists(server, username, password):
"""Check that LDAP user is defined on the LDAP server.
"""
with By("searching LDAP database"):
ldap_node = current().context.cluster.node(server)
r = ldap_node.command(
f"ldapwhoami -H ldap://localhost -D 'cn={user_name},ou=users,dc=company,dc=com' -w {password}"
)
assert r.exitcode == 0, error()
def kube_wait_object(type, name, label="", count=1, ns="test"):
with Then(f"{count} {type}(s) {name} should be created"):
for i in range(1, max_retries):
counts = kube_get_count(type, ns=ns, name=name, label=label)
if counts >= count:
break
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert counts >= count, error()
def decrypt_using_input_table_function(self):
"""Check that we can use `decrypt` function when inserting
data into a table using insert select and `input()` table
function.
"""
node = self.context.node
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader(
"snapshot",
os.path.join(current_dir(), "snapshots",
"insert.py.insert.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}"""
) as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
example_transform = f"decrypt({mode}, unhex(secret), {example_key}{(', ' + example_iv) if example_iv else ''}{(', ' + example_aad) if example_aad else ''})"
with Given("I have ciphertexts"):
example_name = basename(example.name)
ciphertexts = getattr(
snapshot_module,
varname(f"encrypt_input_example_{example_name}"))
example_ciphertexts = [
l.split("\\t")[-1].strip("'")
for l in ciphertexts.split("\\n")
]
with table("user_data"):
with When("I insert decrypted data"):
node.query(
textwrap.dedent(f"""
INSERT INTO
user_data
SELECT
date, name, {example_transform}
FROM
input('date Date, name String, secret String')
FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}')
"""))
with And("I read inserted data back"):
r = node.query(
"SELECT date, name, secret FROM user_data ORDER BY date"
)
expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret"""
with Then("output must match the expected",
description=expected):
assert r.output == expected, error()
def scenario(self, node="clickhouse1"):
"""Check that ClickHouse returns 1 when user executes `SELECT 1` query."""
node = self.context.cluster.node(node)
with When("I execute query select 1"):
r = node.query("SELECT 1").output.strip()
with Then("the result should be 1"):
assert r == "1", error()
def test_metrics_exporter_down(self, prometheus_operator_spec, clickhouse_operator_spec, chi):
def reboot_metrics_exporter():
clickhouse_operator_pod = clickhouse_operator_spec["items"][0]["metadata"]["name"]
kubectl.launch(
f"exec -n {settings.operator_namespace} {clickhouse_operator_pod} -c metrics-exporter -- sh -c 'kill 1'",
ok_to_fail=True,
)
with When("reboot metrics exporter"):
fired = alerts.wait_alert_state(
"ClickHouseMetricsExporterDown", "firing", expected_state=True, callback=reboot_metrics_exporter,
time_range='30s'
)
assert fired, error("can't get ClickHouseMetricsExporterDown alert in firing state")
with Then("check ClickHouseMetricsExporterDown gone away"):
resolved = alerts.wait_alert_state("ClickHouseMetricsExporterDown", "firing", expected_state=False)
assert resolved, error("can't get ClickHouseMetricsExporterDown alert is gone away")
def wait_jsonpath(kind, name, field, value, ns=namespace, retries=max_retries):
with Then(f"{kind} {name} -o jsonpath={field} should be {value}"):
for i in range(1, retries):
cur_value = get_jsonpath(kind, name, field, ns)
if cur_value == value:
break
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert cur_value == value, error()
def wait_object(kind, name, label="", count=1, ns=namespace, retries=max_retries, backoff = 5):
with Then(f"{count} {kind}(s) {name} should be created"):
for i in range(1, retries):
cur_count = get_count(kind, ns=ns, name=name, label=label)
if cur_count >= count:
break
with Then("Not ready. Wait for " + str(i * backoff) + " seconds"):
time.sleep(i * backoff)
assert cur_count >= count, error()
def kube_wait_field(object, name, field, value, ns="test", retries = max_retries):
with Then(f"{object} {name} {field} should be {value}"):
for i in range(1,retries):
obj_status = kubectl(f"get {object} {name} -o=custom-columns=field:{field}", ns=ns).splitlines()
if obj_status[1] == value:
break
with Then("Not ready. Wait for " + str(i*5) + " seconds"):
time.sleep(i*5)
assert obj_status[1] == value, error()
def mismatched_aad(self):
"""Check that `decrypt` function returns garbage or an error when aad parameter does not match."""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
datatype = "String"
plaintext = "'1'"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader(
"snapshot",
os.path.join(current_dir(), "snapshots",
"encrypt.py.encrypt.snapshot"),
).load_module()
for mode, key_len, iv_len, aad_len in modes:
if not aad_len:
continue
with Example(
f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}"""
) as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module,
varname(f"example_{example_name}"))
with When("I decrypt using a mismatched aad"):
r = decrypt(
ciphertext=f"unhex({ciphertext})",
key=f"'{key[:key_len]}'",
mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"),
aad=(None if not aad_len else f"'a{aad}'"),
no_checks=True,
cast="hex",
)
with Then("exitcode shoud be 0 or 198"):
assert r.exitcode in [0, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert ("Exception: Failed to decrypt" in output
or output != "31"), error()
def command(self,
node,
command,
message=None,
exitcode=None,
steps=True,
bash=None,
no_checks=False,
use_error=True,
*args,
**kwargs):
"""Execute and check command.
:param node: name of the service
:param command: command
:param message: expected message that should be in the output, default: None
:param exitcode: expected exitcode, default: None
:param steps: don't break command into steps, default: True
"""
with By("executing command",
description=command,
format_description=False) if steps else NullStep():
if bash is None:
bash = self.bash(node)
try:
r = bash(command, *args, **kwargs)
except ExpectTimeoutError:
self.close_bash(node)
raise
if no_checks:
return r
if exitcode is not None:
with Then(f"exitcode should be {exitcode}",
format_name=False) if steps else NullStep():
assert r.exitcode == exitcode, error(r.output)
if message is not None:
with Then(f"output should contain message",
description=message,
format_description=False) if steps else NullStep():
assert message in r.output, error(r.output)
return r
def dist_table_diff_policies_on_diff_nodes(self, node=None):
"""Check that user is only able to select from the distributed table what is allowed by the row policies on each node.
"""
table_name = f"table_{getuid()}"
dist_table_name = f"dist_table_{getuid()}"
pol_name = f"pol_{getuid()}"
if node is None:
node = self.context.node
node2 = self.context.node2
try:
with Given("I have a table on a cluster"):
node.query(
f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory"
)
with And("I have a row policy"):
node.query(
f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}"
)
with And("I have a distributed table"):
node.query(
f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())"
)
with And("The table has some values on the first node"):
node.query(f"INSERT INTO {table_name} (x) VALUES (1)")
with And("The table has some values on the second node"):
node2.query(f"INSERT INTO {table_name} (x) VALUES (2)")
with When("I alter the row policy to be permissive on the first node"):
node.query(
f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1"
)
with Then("I select from the distributed table"):
output = node.query(f"SELECT * FROM {dist_table_name}").output
assert '1' not in output and '2' in output, error()
finally:
with Finally("I drop the row policy", flags=TE):
node.query(
f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name} ON CLUSTER sharded_cluster"
)
with And("I drop the table", flags=TE):
node.query(
f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster"
)
with And("I drop the distributed table", flags=TE):
node.query(f"DROP TABLE IF EXISTS {dist_table_name}")
def decrypt_using_materialized_view(self):
"""Check that we can use `decrypt` function when inserting
data into a table using a materialized view for input
data transformation.
"""
node = self.context.node
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader(
"snapshot",
os.path.join(current_dir(), "snapshots",
"insert.py.insert.snapshot"),
).load_module()
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}"""
) as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
example_transform = f"decrypt(mode, secret, key{', iv' if example_iv else ''}{', aad' if example_aad else ''})"
with Given("I have ciphertexts"):
example_name = basename(example.name)
ciphertexts = getattr(
snapshot_module,
varname(f"encrypt_mv_example_{example_name}"))
example_ciphertexts = [
"'{}'".format(l.split("\t")[-1].strup("'"))
for l in ciphertexts.split("\n")
]
with table("user_data"):
with mv_transform("user_data", example_transform):
with When("I insert encrypted data"):
node.query(f"""
INSERT INTO user_data_input
(date, name, secret, mode, key)
VALUES
('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}),
('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}),
('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""})
""")
with And("I read inserted data back"):
r = node.query(
"SELECT date, name, secret FROM user_data ORDER BY date"
)
with Then("output must match the expected"):
expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'"""
assert r.output == expected, error()
def nested_live_view_after_policy(self, node=None):
"""Check that if a user has a row policy on a table and a materialized view is created on that table,
the user is only able to select rows specified by the assigned policies from the view.
"""
table_name = f"table_{getuid()}"
view_name = f"view_{getuid()}"
pol_name = f"pol_{getuid()}"
if node is None:
node = self.context.node
with table(node, table_name):
try:
with Given(
"I add allow_experimental_live_view to the default query settings"
):
default_query_settings = getsattr(current().context,
"default_query_settings", [])
default_query_settings.append(
("allow_experimental_live_view", 1))
with And("I have a row policy"):
row_policy(name=pol_name, table=table_name)
with And("The table has some values"):
node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)")
with When("I alter the row policy to be permissive"):
node.query(
f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default"
)
with And("I create a live view on the table"):
node.query(
f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}"
)
with Then("I try to select from the view"):
output = node.query(f"SELECT * FROM {view_name}").output
assert '1' in output and '2' not in output, error()
finally:
with Finally("I drop the live view", flags=TE):
node.query(f"DROP VIEW IF EXISTS {view_name}")
with And(
"I remove allow_experimental_live_view from the default query settings",
flags=TE):
if default_query_settings:
try:
default_query_settings.pop(
default_query_settings.index(
("allow_experimental_live_view", 1)))
except ValueError:
pass
def command(self, node, command, message=None, exitcode=None, steps=True, *args, **kwargs):
"""Execute and check command.
:param node: name of the service
:param command: command
:param message: expected message that should be in the output, default: None
:param exitcode: expected exitcode, default: None
:param steps: don't break command into steps, default: True
"""
debug(f"command() {node}, {command}")
with By("executing command", description=command) if steps else NullStep():
r = self.bash(node)(command, *args, **kwargs)
if exitcode is not None:
with Then(f"exitcode should be {exitcode}") if steps else NullStep():
assert r.exitcode == exitcode, error(r.output)
if message is not None:
with Then(f"output should contain message", description=message) if steps else NullStep():
assert message in r.output, error(r.output)
return r
def test_distributed_files_to_insert():
delayed_pod, delayed_svc, restarted_pod, restarted_svc = random_pod_choice_for_callbacks()
create_distributed_table_on_cluster()
# we need 70 delayed files for catch
insert_sql = 'INSERT INTO default.test_distr(event_time, test) SELECT now(), number FROM system.numbers LIMIT 10000'
# clickhouse.clickhouse_query(
# chi["metadata"]["name"], 'SYSTEM STOP DISTRIBUTED SENDS default.test_distr',
# host=delayed_svc, ns=kubectl.namespace
# )
files_to_insert_from_metrics = 0
files_to_insert_from_disk = 0
tries = 0
while files_to_insert_from_disk < 50 and tries < 500:
kubectl.kubectl(
f"exec -n {kubectl.namespace} {restarted_pod} -c clickhouse -- kill 1",
ok_to_fail=True,
)
clickhouse.clickhouse_query(chi["metadata"]["name"], insert_sql, host=delayed_svc, ns=kubectl.namespace)
files_to_insert_from_metrics = clickhouse.clickhouse_query(
chi["metadata"]["name"], "SELECT value FROM system.metrics WHERE metric='DistributedFilesToInsert'",
host=delayed_svc, ns=kubectl.namespace
)
files_to_insert_from_metrics = int(files_to_insert_from_metrics)
files_to_insert_from_disk = int(kubectl.kubectl(
f"exec -n {kubectl.namespace} {delayed_pod} -c clickhouse -- bash -c 'ls -la /var/lib/clickhouse/data/default/test_distr/*/*.bin 2>/dev/null | wc -l'",
ok_to_fail=False,
))
with When("reboot clickhouse-server pod"):
fired = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", True,
labels={"hostname": delayed_svc, "chi": chi["metadata"]["name"]})
assert fired, error("can't get ClickHouseDistributedFilesToInsertHigh alert in firing state")
# @TODO remove it when https://github.com/ClickHouse/ClickHouse/pull/11220 will merged to docker latest image
kubectl.kube_wait_pod_status(restarted_pod, "Running", ns=kubectl.namespace)
with Then("check ClickHouseClickHouseDistributedFilesToInsertHigh gone away"):
resolved = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", False, labels={"hostname": delayed_svc})
assert resolved, error("can't check ClickHouseDistributedFilesToInsertHigh alert is gone away")
drop_distributed_table_on_cluster()
def on_cluster(self, node=None):
"""Check that a row policy created using ON CLUSTER applies to the nodes of the cluster correctly."""
table_name = f"table_{getuid()}"
pol_name = f"pol_{getuid()}"
if node is None:
node = self.context.node
node2 = self.context.node2
try:
with Given("I have a table on a cluster"):
node.query(
f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory"
)
with And("I have a row policy"):
node.query(
f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1"
)
with When("I insert some values into the table on the first node"):
node.query(f"INSERT INTO {table_name} (x) VALUES (1)")
with And("I insert some values into the table on the second node"):
node2.query(f"INSERT INTO {table_name} (x) VALUES (1)")
with Then("I select from the table"):
output = node.query(f"SELECT * FROM {table_name}").output
assert "" == output, error()
with And("I select from another node on the cluster"):
output = node2.query(f"SELECT * FROM {table_name}").output
assert "" == output, error()
finally:
with Finally("I drop the row policy", flags=TE):
node.query(
f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}"
)
with And("I drop the table", flags=TE):
node.query(f"DROP TABLE {table_name} ON CLUSTER sharded_cluster")
def cmd(self,
cmd,
message=None,
exitcode=None,
steps=True,
shell_command="bash --noediting",
no_checks=False,
raise_on_exception=False,
step=By,
*args,
**kwargs):
"""Execute and check command.
:param cmd: command
:param message: expected message that should be in the output, default: None
:param exitcode: expected exitcode, default: None
"""
command = f"{cmd}"
with step("executing command",
description=command,
format_description=False) if steps else NullStep():
try:
r = self.cluster.bash(self.name,
command=shell_command)(command, *args,
**kwargs)
except ExpectTimeoutError:
self.cluster.close_bash(self.name)
raise
if no_checks:
return r
if exitcode is not None:
with Then(
f"exitcode should be {exitcode}") if steps else NullStep():
assert r.exitcode == exitcode, error(r.output)
if message is not None:
with Then(f"output should contain message",
description=message) if steps else NullStep():
assert message in r.output, error(r.output)
return r
def sanity_check(self, connection):
"""Check connection to the database.
"""
with Given("PyODBC connection"):
with When("I do 'SELECT 1'"):
rows = connection.query("SELECT 1")
result = "[(1, )]"
with Then(f"the result is {result}", format_name=False):
assert repr(rows) == result, error("result dit not match")
def mismatched_iv(self):
"""Check that `aes_decrypt_mysql` function returns garbage or an error when iv parameter does not match."""
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader(
"snapshot",
os.path.join(
current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot"
),
).load_module()
for mode, key_len, iv_len in mysql_modes:
if not iv_len:
continue
with Example(
f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}"""
) as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(
snapshot_module, varname(f"example_{example_name}")
)
with When("I decrypt using a mismatched key"):
r = aes_decrypt_mysql(
ciphertext=f"unhex({ciphertext})",
key=f"'{key[:key_len]}'",
mode=mode,
iv=f"'a{iv[:iv_len-1]}'",
cast="hex",
no_checks=True,
)
with Then("exitcode shoud be 0 or 198"):
assert r.exitcode in [0, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert (
"Exception: Failed to decrypt" in output or output != "31"
), error()
def check_ttl_when_privilege_is_granted(table, user, node):
"""Ensures ALTER TTL runs as expected when the privilege is granted to the specified user
"""
with Given(f"I modify TTL"):
node.query(f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 1 DAY;",
settings = [("user", user)])
with Then("I verify that the TTL clause is in the table"):
output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output)
assert "TTL d + toIntervalDay(1)" in output['statement'], error()
def test_too_many_connections():
too_many_connection_pod, too_many_connection_svc, _, _ = random_pod_choice_for_callbacks()
cmd = "export DEBIAN_FRONTEND=noninteractive; apt-get update; apt-get install -y netcat mysql-client"
kubectl.launch(
f"exec -n {kubectl.namespace} {too_many_connection_pod} -c clickhouse -- bash -c \"{cmd}\"",
)
def make_too_many_connection():
long_cmd = ""
for _ in range(120):
port = random.choice(["8123", "3306", "9000"])
if port == "8123":
# HTTPConnection metric increase after full parsing of HTTP Request, we can't provide pause between CONNECT and QUERY running
# long_cmd += f"nc -vv 127.0.0.1 {port} <( printf \"POST / HTTP/1.1\\r\\nHost: 127.0.0.1:8123\\r\\nContent-Length: 34\\r\\n\\r\\nTEST\\r\\nTEST\\r\\nTEST\\r\\nTEST\\r\\nTEST\\r\\nTEST\");"
long_cmd += 'wget -qO- "http://127.0.0.1:8123?query=SELECT sleepEachRow(1),number,now() FROM numbers(30)";'
elif port == "9000":
long_cmd += 'clickhouse-client --idle_connection_timeout 70 --receive_timeout 70 -q "SELECT sleepEachRow(1),number,now() FROM numbers(30)";'
# elif port == "3306":
# long_cmd += 'mysql -u default -h 127.0.0.1 -e "SELECT sleepEachRow(1),number, now() FROM numbers(30)";'
else:
long_cmd += f"printf \"1\\n1\" | nc -q 5 -i 30 -vv 127.0.0.1 {port};"
nc_cmd = f"echo '{long_cmd} exit 0' | xargs --verbose -i'{{}}' --no-run-if-empty -d ';' -P 120 bash -c '{{}}' 1>/dev/null"
with open("/tmp/nc_cmd.sh", "w") as f:
f.write(nc_cmd)
kubectl.launch(
f"cp /tmp/nc_cmd.sh {too_many_connection_pod}:/tmp/nc_cmd.sh -c clickhouse"
)
kubectl.launch(
f"exec -n {kubectl.namespace} {too_many_connection_pod} -c clickhouse -- bash /tmp/nc_cmd.sh",
timeout=600,
)
with Then("check ClickHouseTooManyConnections firing"):
fired = wait_alert_state("ClickHouseTooManyConnections", "firing", True, labels={"hostname": too_many_connection_svc},
time_range='90s', callback=make_too_many_connection)
assert fired, error("can't get ClickHouseTooManyConnections alert in firing state")
with Then("check ClickHouseTooManyConnections gone away"):
resolved = wait_alert_state("ClickHouseTooManyConnections", "firing", False, labels={"hostname": too_many_connection_svc})
assert resolved, error("can't check ClickHouseTooManyConnections alert is gone away")
def get_minio_spec():
with Given("get information about prometheus installation"):
minio_spec = kubectl.get("pod",
ns=settings.minio_namespace,
name="",
label="-l app=minio")
assert "items" in minio_spec and len(
minio_spec["items"]) and "metadata" in minio_spec["items"][
0], error("invalid minio spec, please run install-minio.sh")
return minio_spec
def check_alter_settings_when_privilege_is_granted(table, user, node):
"""Ensures ADD SETTINGS runs as expected when the privilege is granted to the specified user
"""
with Given(
"I check that the modified setting is not already in the table"):
output = json.loads(
node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output)
assert "merge_with_ttl_timeout = 5" not in output['statement'], error()
with And(f"I modify settings"):
node.query(
f"ALTER TABLE {table} MODIFY SETTING merge_with_ttl_timeout=5",
settings=[("user", user)])
with Then("I verify that the setting is in the table"):
output = json.loads(
node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output)
assert "SETTINGS index_granularity = 8192, merge_with_ttl_timeout = 5" in output[
'statement'], error()
def setup(self, arg):
with Given("I setup something"):
pass
assert self.parent.name == "/setup parameter/test", error()
yield
with By("cleaning up something"):
pass
def add_user_to_ldap(cn,
userpassword,
givenname=None,
homedirectory=None,
sn=None,
uid=None,
uidnumber=None,
node=None):
"""Add user entry to LDAP."""
if node is None:
node = current().context.ldap_node
if uid is None:
uid = cn
if givenname is None:
givenname = "John"
if homedirectory is None:
homedirectory = "/home/users"
if sn is None:
sn = "User"
if uidnumber is None:
uidnumber = 2000
user = {
"dn": f"cn={cn},ou=users,dc=company,dc=com",
"cn": cn,
"gidnumber": 501,
"givenname": givenname,
"homedirectory": homedirectory,
"objectclass": ["inetOrgPerson", "posixAccount", "top"],
"sn": sn,
"uid": uid,
"uidnumber": uidnumber,
"userpassword": userpassword,
"_server": node.name
}
lines = []
for key, value in list(user.items()):
if key.startswith("_"):
continue
elif key == "objectclass":
for cls in value:
lines.append(f"objectclass: {cls}")
else:
lines.append(f"{key}: {value}")
ldif = "\n".join(lines)
r = node.command(
f"echo -e \"{ldif}\" | ldapadd -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin"
)
assert r.exitcode == 0, error()
return user
def decrypt_multiple(self, count=1000):
"""Check decrypting column when reading multiple entries
encrypted with the same parameters for the same user
from a table.
"""
node = self.context.node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
aad = "some random aad"
for mode, key_len, iv_len, aad_len in modes:
with Example(
f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}"""
) as example:
with table(
"user_table",
"""
CREATE TABLE {name}
(
date Nullable(Date),
name Nullable(String),
secret Nullable(String)
)
ENGINE = Memory()
""",
):
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
with When("I insert encrypted data"):
encrypted_secret = node.query(
f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))"""
).output.strip()
values = [
f"('2020-01-01', 'user0', unhex('{encrypted_secret}'))"
] * count
node.query("INSERT INTO user_table\n"
" (date, name, secret)\n"
f"VALUES {', '.join(values)}")
with And(
"I decrypt data",
description=
"using a subquery and get the number of entries that match the plaintext",
):
output = node.query(
f"""SELECT count() AS count FROM (SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table) WHERE secret = 'secret' FORMAT JSONEachRow"""
).output.strip()
with Then("I should get back the expected result",
description=f"{count}"):
assert output == f'{{"count":"{count}"}}', error()
def role_column_privileges(self,
grant_columns,
select_columns_pass,
data_pass,
table_type,
revoke_columns=None,
select_columns_fail=None,
node=None):
"""Check that user is able to select from granted columns and unable
to select from not granted or revoked columns.
"""
user_name = f"user_{getuid()}"
role_name = f"role_{getuid()}"
table_name = f"table_{getuid()}"
if node is None:
node = self.context.node
with table(node, table_name, table_type):
with Given("The table has some data on some columns"):
node.query(
f"INSERT INTO {table_name} ({select_columns_pass}) VALUES ({data_pass})"
)
with user(node, user_name), role(node, role_name):
with When("I grant select privilege"):
node.query(
f"GRANT SELECT({grant_columns}) ON {table_name} TO {role_name}"
)
with And("I grant the role to a user"):
node.query(f"GRANT {role_name} TO {user_name}")
if select_columns_fail is not None:
with And("I select from not granted column"):
exitcode, message = errors.not_enough_privileges(
name=user_name)
node.query(
f"SELECT ({select_columns_fail}) FROM {table_name}",
settings=[("user", user_name)],
exitcode=exitcode,
message=message)
with Then("I verify SELECT command"):
user_select = node.query(f"SELECT d FROM {table_name}",
settings=[("user", user_name)])
default = node.query(f"SELECT d FROM {table_name}")
assert user_select.output == default.output, error()
if revoke_columns is not None:
with When("I revoke select privilege for columns from role"):
node.query(
f"REVOKE SELECT({revoke_columns}) ON {table_name} FROM {role_name}"
)
with And("I select from revoked columns"):
exitcode, message = errors.not_enough_privileges(
name=user_name)
node.query(
f"SELECT ({select_columns_pass}) FROM {table_name}",
settings=[("user", user_name)],
exitcode=exitcode,
message=message)