def test_users_have_appropriate_permissions(hdfs_client, kerberos):
# "hdfs" is a superuser
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")
)
alice_dir = "/users/alice"
test_filename = config.check_test_users_have_appropriate_permissions(alice_dir)
# alice has read/write access to her directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("alice")
)
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
_, stdout, _ = config.hdfs_client_list_files(alice_dir)
assert test_filename in stdout
# bob doesn't have read/write access to alice's directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("bob"))
config.hdfs_client_write_data(
test_filename, expect_failure_message="put: Permission denied: user=bob"
)
config.hdfs_client_read_data(
test_filename, expect_failure_message="cat: Permission denied: user=bob"
)
def test_users_have_appropriate_permissions(hdfs_client, kerberos):
# "hdfs" is a superuser
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")
)
alice_dir = "/users/alice"
config.run_client_command(" && ".join([
config.hdfs_command(c) for c in [
"mkdir -p {}".format(alice_dir),
"chown alice:users {}".format(alice_dir),
"chmod 700 {}".format(alice_dir),
]
]))
test_filename = "{}/{}".format(alice_dir, config.get_unique_filename("test_kerberos_auth_user_permissions"))
# alice has read/write access to her directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("alice")
)
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
_, stdout, _ = config.hdfs_client_list_files(alice_dir)
assert test_filename in stdout
# bob doesn't have read/write access to alice's directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("bob"))
config.hdfs_client_write_data(test_filename, expect_failure_message="put: Permission denied: user=bob")
config.hdfs_client_read_data(test_filename, expect_failure_message="cat: Permission denied: user=bob")
def test_users_have_appropriate_permissions(hdfs_client, kerberos):
# "hdfs" is a superuser
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")
)
alice_dir = "/users/alice"
config.run_client_command(" && ".join([
config.hdfs_command(c) for c in [
"mkdir -p {}".format(alice_dir),
"chown alice:users {}".format(alice_dir),
"chmod 700 {}".format(alice_dir),
]
]))
test_filename = "{}/{}".format(alice_dir, config.get_unique_filename("test_ssl_kerberos_auth_user_permissions"))
# alice has read/write access to her directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("alice")
)
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
_, stdout, _ = config.hdfs_client_list_files(alice_dir)
assert test_filename in stdout
# bob doesn't have read/write access to alice's directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("bob"))
config.hdfs_client_write_data(test_filename, expect_failure_message="put: Permission denied: user=bob")
config.hdfs_client_read_data(test_filename, expect_failure_message="cat: Permission denied: user=bob")
def test_user_can_auth_and_write_and_read(hdfs_client, kerberos):
sdk_auth.kinit(hdfs_client["id"],
keytab=config.KEYTAB,
principal=kerberos.get_principal("hdfs"))
test_filename = config.get_unique_filename("test_kerberos_auth_write_read")
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
def test_user_can_auth_and_write_and_read(hdfs_client, kerberos):
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")
)
test_filename = config.get_unique_filename("test_ssl_kerberos_auth_write_read")
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
def test_users_have_appropriate_permissions(hdfs_client, kerberos):
# "hdfs" is a superuser
sdk_auth.kinit(hdfs_client["id"],
keytab=config.KEYTAB,
principal=kerberos.get_principal("hdfs"))
alice_dir = "/users/alice"
success = True
cmd_lists = [
["mkdir", "-p", alice_dir],
["chown", "alice:users", alice_dir],
["chmod", "700", alice_dir],
]
for cmd_list in cmd_lists:
cmd = config.hdfs_command(" ".join(cmd_list))
cmd_success = config.run_client_command(cmd)
if not cmd_success:
log.error("Error executing: %s", cmd)
success = success and cmd_success
if not success:
log.error("Error creating %s", alice_dir)
raise Exception("Error creating user directory")
# alice has read/write access to her directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(hdfs_client["id"],
keytab=config.KEYTAB,
principal=kerberos.get_principal("alice"))
test_filename = "{}/{}".format(
alice_dir,
config.get_unique_filename("test_kerberos_auth_user_permissions"))
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
_, stdout, _ = config.hdfs_client_list_files(alice_dir)
assert test_filename in stdout
# bob doesn't have read/write access to alice's directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(hdfs_client["id"],
keytab=config.KEYTAB,
principal=kerberos.get_principal("bob"))
config.hdfs_client_write_data(
test_filename,
expect_failure_message="put: Permission denied: user=bob")
config.hdfs_client_read_data(
test_filename,
expect_failure_message="cat: Permission denied: user=bob")
def test_integrity_on_data_node_failure(hdfs_client):
"""
Verifies proper data replication among data nodes.
"""
test_filename = config.get_unique_filename("test_datanode_fail")
# An HDFS write will only successfully return when the data replication has taken place
config.hdfs_client_write_data(test_filename)
# Should have 3 data nodes (data-0,1,2), kill 2 of them:
data_tasks = sdk_tasks.get_service_tasks(config.SERVICE_NAME, "data")
for idx in range(2):
sdk_cmd.kill_task_with_pattern("DataNode", "nobody", agent_host=data_tasks[idx].host)
config.hdfs_client_read_data(test_filename)
config.check_healthy(config.SERVICE_NAME)
def test_integrity_on_name_node_failure(hdfs_client):
"""
The first name node (name-0-node) is the active name node by default when HDFS gets installed.
This test checks that it is possible to write and read data after the active name node fails
so as to verify a failover sustains expected functionality.
"""
@retrying.retry(
wait_fixed=1000,
stop_max_delay=config.DEFAULT_HDFS_TIMEOUT * 1000
)
def _get_active_name_node():
for candidate in ("name-0-node", "name-1-node"):
if is_name_node_active(candidate):
return candidate
raise Exception("Failed to determine active name node")
active_name_node = _get_active_name_node()
sdk_cmd.kill_task_with_pattern(
"NameNode",
"nobody",
agent_host=sdk_tasks.get_service_tasks(config.SERVICE_NAME, active_name_node)[0].host,
)
# After the previous active namenode was killed, the opposite namenode should marked active:
if active_name_node == "name-1-node":
new_active_name_node = "name-0-node"
else:
new_active_name_node = "name-1-node"
@retrying.retry(
wait_fixed=1000,
stop_max_delay=config.DEFAULT_HDFS_TIMEOUT * 1000,
retry_on_result=lambda res: not res,
)
def _wait_for_failover_to_complete(namenode):
return is_name_node_active(namenode)
_wait_for_failover_to_complete(new_active_name_node)
test_filename = config.get_unique_filename("test_namenode_fail")
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
config.check_healthy(config.SERVICE_NAME)
def test_users_have_appropriate_permissions(hdfs_client, kerberos):
# "hdfs" is a superuser
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")
)
alice_dir = "/users/alice"
success = True
cmd_lists = [
["mkdir", "-p", alice_dir],
["chown", "alice:users", alice_dir],
["chmod", "700", alice_dir],
]
for cmd_list in cmd_lists:
cmd = config.hdfs_command(" ".join(cmd_list))
cmd_success = config.run_client_command(cmd)
if not cmd_success:
log.error("Error executing: %s", cmd)
success = success and cmd_success
if not success:
log.error("Error creating %s", alice_dir)
raise Exception("Error creating user directory")
# alice has read/write access to her directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(
hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("alice")
)
test_filename = "{}/{}".format(alice_dir, config.get_unique_filename("test_kerberos_auth_user_permissions"))
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
_, stdout, _ = config.hdfs_client_list_files(alice_dir)
assert test_filename in stdout
# bob doesn't have read/write access to alice's directory
sdk_auth.kdestroy(hdfs_client["id"])
sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("bob"))
config.hdfs_client_write_data(test_filename, expect_failure_message="put: Permission denied: user=bob")
config.hdfs_client_read_data(test_filename, expect_failure_message="cat: Permission denied: user=bob")
def test_write_and_read_data_on_overlay(hdfs_client):
test_filename = config.get_unique_filename("test_data_overlay")
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)
config.check_healthy(config.SERVICE_NAME)
def test_write_and_read_data_over_tls(hdfs_service, hdfs_client):
test_filename = config.get_unique_filename("test_data_tls")
config.hdfs_client_write_data(test_filename)
config.hdfs_client_read_data(test_filename)