def test_select_auth_no_match(auth_config, tmpdir):
no_match = "No matching credential found"
# no credentials at all
with pytest.raises(RuntimeError) as err:
selected = auth.select_matching_auth([], "example.com", "nosuchuser")
assert "No matching credential found for hostname 'example.com'" in err.value.args[
0]
# no match for requested hostname
with_host = """auth = [{
username = "******",
password = "******",
hostname = "example.com"
}]"""
with temp_config(tmpdir, with_host) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(RuntimeError) as err:
creds = auth.load_auth()
selected = auth.select_matching_auth(creds, "example.net")
assert f"{no_match} for hostname 'example.net'" in err.value.args[0]
# no match for requested username
with temp_config(tmpdir, auth_config) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(RuntimeError) as err:
creds = auth.load_auth()
selected = auth.select_matching_auth(creds, "example.com",
"nosuchuser")
assert f"{no_match} for hostname 'example.com' with username 'nosuchuser'" in err.value.args[
0]
def test_load_auth_options(auth_config, tmpdir):
# SSL should be used by default
# The default mechanism should be SCRAM_SHA_512
with temp_config(tmpdir, auth_config) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(ssl=True)
from adc.auth import SASLMethod
assert auth_mock.called_with(mechanism=SASLMethod.SCRAM_SHA_512)
# But it should be possible to disable SSL
use_plaintext = """auth = [{
username = "******",
password = "******",
protocol = "SASL_PLAINTEXT"
}]"""
with temp_config(tmpdir, use_plaintext) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(ssl=False)
# An SSL CA data path should be honored
with_ca_data = """auth = [{
username = "******",
password = "******",
ssl_ca_location = "/foo/bar/baz"
}]"""
with temp_config(tmpdir, with_ca_data) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(ssl_ca_location="/foo/bar/baz")
# Alternate mechanisms should be honored
plain_mechanism = """auth = [{
username = "******",
password = "******",
mechanism = "PLAIN"
}]"""
with temp_config(tmpdir, plain_mechanism) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(mechanism=SASLMethod.PLAIN)
# Associated hostnames should be included
with_host = """auth = [{
username = "******",
password = "******",
hostname = "example.com"
}]"""
with temp_config(tmpdir, with_host) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir):
creds = auth.load_auth()
assert len(creds) == 1
assert creds[0].hostname == "example.com"
def test_load_auth_malformed(tmpdir):
missing_username = """
auth = [{extra="stuff",
password="******"}]
"""
with temp_config(tmpdir, missing_username) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(RuntimeError):
auth.load_auth()
missing_password = """
auth = [{username="******",
extra="stuff"}]
"""
with temp_config(tmpdir, missing_password) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(RuntimeError):
auth.load_auth()
def test_list_credentials(script_runner, auth_config, tmpdir):
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("hop", "auth", "list")
assert ret.success
assert "username" in ret.stdout
assert ret.stderr == ""
def test_stream_open(auth_config, tmpdir):
stream = io.Stream(auth=False)
# verify only read/writes are allowed
with pytest.raises(ValueError) as err:
stream.open("kafka://localhost:9092/topic1", "q")
assert "mode must be either 'w' or 'r'" in err.value.args
# verify that URLs with no scheme are rejected
with pytest.raises(ValueError) as err:
stream.open("bad://example.com/topic", "r")
assert "invalid kafka URL: must start with 'kafka://'" in err.value.args
# verify that URLs with no topic are rejected
with pytest.raises(ValueError) as err:
stream.open("kafka://example.com/", "r")
assert "no topic(s) specified in kafka URL" in err.value.args
# verify that complete URLs are accepted
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(XDG_CONFIG_HOME=config_dir), \
patch("adc.consumer.Consumer.subscribe", MagicMock()) as subscribe:
stream = io.Stream()
# opening a valid URL for reading should succeed
consumer = stream.open("kafka://example.com/topic", "r")
# an appropriate consumer group name should be derived from the username in the auth
assert consumer._consumer.conf.group_id.startswith(
stream.auth.username)
# the target topic should be subscribed to
subscribe.assert_called_once_with("topic")
# opening a valid URL for writing should succeed
producer = stream.open("kafka://example.com/topic", "w")
producer.write("data")
def test_load_auth_bad_perms(auth_config, tmpdir):
for bad_perm in [
stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH,
stat.S_IWOTH, stat.S_IXOTH
]:
with temp_config(tmpdir, auth_config, bad_perm) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(RuntimeError):
auth.load_auth()
def test_config_advice(script_runner, auth_config, tmpdir):
advice_tag = "No valid credential data found"
# nonexistent config file
with temp_environ(XDG_CONFIG_HOME=str(tmpdir)):
ret = script_runner.run("hop")
assert advice_tag in ret.stdout
# wrong credential file permissions
import stat
with temp_config(tmpdir, "", stat.S_IROTH) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("hop")
assert advice_tag in ret.stdout
assert "unsafe permissions" in ret.stderr
# syntactically invalid TOML in credential file
garbage = "JVfwteouh '652b"
with temp_config(
tmpdir,
garbage) as config_dir, temp_environ(XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("hop")
assert advice_tag in ret.stdout
assert "not configured correctly" in ret.stderr
# syntactically valid TOML without an [auth] section
toml_no_auth = """title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
"""
with temp_config(tmpdir, toml_no_auth) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("hop")
assert advice_tag in ret.stdout
assert "configuration file has no auth section" in ret.stderr
# syntactically valid TOML an incomplete [auth] section
toml_bad_auth = """[auth]
foo = "bar"
"""
with temp_config(tmpdir, toml_bad_auth) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("hop")
assert advice_tag in ret.stdout
assert "missing auth property" in ret.stderr
def test_load_auth_options(auth_config, tmpdir):
# SSL should be used by default
# The default mechanism should be SCRAM_SHA_512
with temp_config(tmpdir, auth_config) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(ssl=True)
from adc.auth import SASLMethod
assert auth_mock.called_with(mechanism=SASLMethod.SCRAM_SHA_512)
# But it should be possible to disable SSL
use_plaintext = """
[auth]
username = "******"
password = "******"
protocol = "SASL_PLAINTEXT"
"""
with temp_config(tmpdir, use_plaintext) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(ssl=False)
# An SSL CA data path should be honored
with_ca_data = """
[auth]
username = "******"
password = "******"
ssl_ca_location = "/foo/bar/baz"
"""
with temp_config(tmpdir, with_ca_data) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(ssl_ca_location="/foo/bar/baz")
# Alternate mechanisms should be honored
plain_mechanism = """
[auth]
username = "******"
password = "******"
mechanism = "PLAIN"
"""
with temp_config(tmpdir, plain_mechanism) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), patch("hop.auth.Auth") as auth_mock:
auth.load_auth()
assert auth_mock.called_with(mechanism=SASLMethod.PLAIN)
def test_load_auth_malformed(tmpdir):
missing_username = """
[auth]
password = "******"
extra = "stuff"
"""
with temp_config(tmpdir, missing_username) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(KeyError):
auth.load_auth()
missing_password = """
[auth]
username = "******"
extra = "stuff"
"""
with temp_config(tmpdir, missing_password) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(KeyError):
auth.load_auth()
def test_add_credential(script_runner, auth_config, tmpdir):
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
csv_file = str(tmpdir) + "/new_cred.csv"
with open(csv_file, "w") as f:
f.write("username,password\nnew_user,new_pass")
ret = script_runner.run("hop", "auth", "add", csv_file)
assert ret.success
assert "Wrote configuration to" in ret.stderr
def test_cli_version(script_runner, auth_config, tmpdir):
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("hop", "version", "--help")
assert ret.success
assert ret.stderr == ""
ret = script_runner.run("hop", "version")
assert ret.success
assert f"hop-client=={__version__}\n" in ret.stdout
assert ret.stderr == ""
def test_cli_auth(script_runner, auth_config, tmpdir):
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
ret1 = script_runner.run("hop", "auth", "--help")
assert ret1.success
assert ret1.stderr == ""
ret = script_runner.run("hop", "auth", "locate")
assert ret.success
assert config_dir in ret.stdout
assert ret.stderr == ""
def test_cli_hop_module(script_runner, auth_config, tmpdir):
ret = script_runner.run("python", "-m", "hop", "--help")
assert ret.success
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("python", "-m", "hop", "--version")
assert ret.success
assert f"hop version {__version__}\n" in ret.stdout
assert ret.stderr == ""
def test_add_credential_to_nonempty(auth_config, tmpdir):
old_cred = auth.Auth("username", "password")
new_cred = auth.Auth("other_user", "other_pass")
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(HOME=config_dir), \
patch("hop.auth.read_new_credential", MagicMock(return_value=new_cred)):
args = MagicMock()
args.cred_file = None
args.force = False
auth.add_credential(args)
check_credential_file(configure.get_config_path("auth"), old_cred)
check_credential_file(configure.get_config_path("auth"), new_cred)
def test_add_credential_overwrite(script_runner, auth_config, tmpdir):
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
csv_file = str(tmpdir) + "/new_cred.csv"
with open(csv_file, "w") as f:
f.write("username,password\nnew_user,new_pass")
ret = script_runner.run("hop", "auth", "add", csv_file)
assert ret.success
assert "Wrote configuration to" in ret.stderr
with open(csv_file, "w") as f:
f.write("username,password\nnew_user,other_pass")
# try to overwrite the credential, without forcing
ret = script_runner.run("hop", "auth", "add", csv_file)
assert ret.success
assert "Credential already exists; overwrite with --force" in ret.stderr
# try again, with force
ret = script_runner.run("hop", "auth", "add", "--force", csv_file)
assert ret.success
assert "Wrote configuration to" in ret.stderr
def test_stream_auth(auth_config, tmpdir):
# turning off authentication should give None for the auth property
s1 = io.Stream(auth=False)
assert s1.auth is None
# turning on authentication should give an auth object with the data read from the default file
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
s2 = io.Stream(auth=True)
a2 = s2.auth
assert a2._config["sasl.username"] == "username"
assert a2._config["sasl.password"] == "password"
assert a2.username == "username"
# turning on authentication should fail when the default file does not exist
with temp_environ(
XDG_CONFIG_HOME=str(tmpdir)), pytest.raises(FileNotFoundError):
s3 = io.Stream(auth=True)
a3 = s3.auth
# anything other than a bool passed to the Stream constructor should get handed back unchanged
s4 = io.Stream(auth="blarg")
assert s4.auth == "blarg"
def test_load_auth_muliple_creds(tmpdir):
two_creds = """auth = [
{
username = "******",
password = "******",
hostname = "host1"
},
{
username = "******",
password = "******",
hostname = "host2"
},
]"""
with temp_config(
tmpdir,
two_creds) as config_dir, temp_environ(XDG_CONFIG_HOME=config_dir):
creds = auth.load_auth()
assert len(creds) == 2
assert creds[0].username == "user1"
assert creds[0].password == "pass1"
assert creds[0].hostname == "host1"
assert creds[1].username == "user2"
assert creds[1].password == "pass2"
assert creds[1].hostname == "host2"
def test_load_auth_invalid_toml(tmpdir):
garbage = "KHFBGKJSBVJKbdfb ,s ,msb vks bs"
with temp_config(tmpdir, garbage) as config_dir, \
temp_environ(XDG_CONFIG_HOME=config_dir), pytest.raises(RuntimeError):
auth.load_auth()
def test_load_auth(auth_config, tmpdir):
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
auth_data = auth.load_auth()
assert len(auth_data) == 1
assert auth_data[0].username == "username"
def test_delete_credential(script_runner, auth_config, tmpdir):
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(
XDG_CONFIG_HOME=config_dir):
ret = script_runner.run("hop", "auth", "remove", "username")
assert ret.success
assert "Wrote configuration to" in ret.stderr
def test_cli_list_topics(script_runner, auth_config, tmpdir):
ret = script_runner.run("hop", "list-topics", "--help")
assert ret.success
broker_url = "kafka://hostname:port/"
# general listing when no topics are returned
with patch("confluent_kafka.Consumer",
make_consumer_mock({})) as mock_consumer:
ret = script_runner.run("hop", "list-topics", broker_url, "--no-auth")
assert ret.success
assert ret.stderr == ""
assert "No accessible topics" in ret.stdout
mock_consumer.assert_called()
mock_consumer.return_value.list_topics.assert_called_with()
expected_topics = ["foo", "bar"]
unexpected_topics = ["baz"]
topic_results = {}
for topic in expected_topics:
topic_results[topic] = dummy_topic_info(topic)
for topic in unexpected_topics:
topic_results[topic] = dummy_topic_info(topic, "an error")
# general listing when some topics are returned
with patch("confluent_kafka.Consumer",
make_consumer_mock(topic_results)) as mock_consumer:
ret = script_runner.run("hop", "--debug", "list-topics", broker_url,
"--no-auth")
assert ret.success
assert ret.stderr == ""
assert "Accessible topics" in ret.stdout
for topic in expected_topics:
assert topic in ret.stdout
for topic in unexpected_topics:
assert topic not in ret.stdout
mock_consumer.assert_called()
mock_consumer.return_value.list_topics.assert_called_with()
query_topics = ["foo", "bar", "baz"]
# listing of specific topics, none of which exist
with patch("confluent_kafka.Consumer",
make_consumer_mock({})) as mock_consumer:
ret = script_runner.run("hop", "list-topics",
broker_url + ",".join(query_topics),
"--no-auth")
assert ret.success
assert ret.stderr == ""
assert "No accessible topics" in ret.stdout
mock_consumer.assert_called()
for topic in query_topics:
mock_consumer.return_value.list_topics.assert_any_call(topic=topic)
# listing of specific topics, some of which exist and some of which do not
with patch("confluent_kafka.Consumer",
make_consumer_mock(topic_results)) as mock_consumer:
ret = script_runner.run("hop", "list-topics",
broker_url + ",".join(query_topics),
"--no-auth")
assert ret.success
assert ret.stderr == ""
assert "Accessible topics" in ret.stdout
for topic in expected_topics:
assert topic in ret.stdout
for topic in unexpected_topics:
assert topic not in ret.stdout
mock_consumer.assert_called()
for topic in query_topics:
mock_consumer.return_value.list_topics.assert_any_call(topic=topic)
# general listing with authentication
with temp_config(tmpdir, auth_config) as config_dir, temp_environ(XDG_CONFIG_HOME=config_dir), \
patch("confluent_kafka.Consumer", make_consumer_mock(topic_results)) as mock_consumer:
ret = script_runner.run("hop", "list-topics", broker_url)
assert ret.success
assert ret.stderr == ""
assert "Accessible topics" in ret.stdout
for topic in expected_topics:
assert topic in ret.stdout
for topic in unexpected_topics:
assert topic not in ret.stdout
mock_consumer.assert_called()
mock_consumer.return_value.list_topics.assert_called_with()
# attempting to use multiple brokers should provoke an error
ret = script_runner.run("hop", "list-topics",
"kafka://example.com,example.net")
assert not ret.success
assert "Multiple broker addresses are not supported" in ret.stderr
