def verify_running():
sys.stderr.write("running; ")
@inlineCallbacks
def validate_connected(client):
self.assertTrue(client.connected)
sys.stderr.write("connected.")
exists_deferred, watch_deferred = client.exists_and_watch(
"/charms")
stat = yield exists_deferred
if stat:
test_complete.callback(None)
returnValue(True)
yield watch_deferred
stat = yield client.exists("/charms")
self.assertTrue(stat)
test_complete.callback(None)
def propogate_failure(failure):
test_complete.errback(failure)
return failure
def close_client(result, client):
client.close()
server = "%s:2181" % instances[0].dns_name
client = SSHClient()
client_deferred = client.connect(server, timeout=300)
client_deferred.addCallback(validate_connected)
client_deferred.addErrback(propogate_failure)
client_deferred.addBoth(close_client, client)
def verify_running():
sys.stderr.write("running; ")
@inlineCallbacks
def validate_connected(client):
self.assertTrue(client.connected)
sys.stderr.write("connected.")
exists_deferred, watch_deferred = client.exists_and_watch(
"/charms")
stat = yield exists_deferred
if stat:
test_complete.callback(None)
returnValue(True)
yield watch_deferred
stat = yield client.exists("/charms")
self.assertTrue(stat)
test_complete.callback(None)
def propogate_failure(failure):
test_complete.errback(failure)
return failure
def close_client(result, client):
client.close()
server = "%s:2181" % instances[0].dns_name
client = SSHClient()
client_deferred = client.connect(server, timeout=300)
client_deferred.addCallback(validate_connected)
client_deferred.addErrback(propogate_failure)
client_deferred.addBoth(close_client, client)
def setUp(self):
super(ConnectionTest, self).setUp()
self.username = pwd.getpwuid(os.getuid())[0]
self.log = self.capture_logging("juju.state.sshforward")
self.old_user_name = SSHClient.remote_user
SSHClient.remote_user = self.username
self.client = SSHClient()
zookeeper.set_debug_level(0)
def setUp(self):
super(ConnectionTest, self).setUp()
self.username = pwd.getpwuid(os.getuid())[0]
self.log = self.capture_logging("juju.state.sshforward")
self.old_user_name = SSHClient.remote_user
SSHClient.remote_user = self.username
self.client = SSHClient()
zookeeper.set_debug_level(0)
def _internal_connect(self, share):
"""Attempt connection to one of the ZK nodes."""
candidates = yield self._provider.get_zookeeper_machines()
assigned = [machine for machine in candidates if machine.dns_name]
if not assigned:
yield sleep(1) # Momentarily backoff
raise EnvironmentPending("No machines have assigned addresses")
chosen = random.choice(assigned)
log.debug("Connecting to environment using %s...", chosen.dns_name)
try:
client = yield SSHClient().connect(
chosen.dns_name + ":2181", timeout=30, share=share)
except (NoConnection, ConnectionTimeoutException) as e:
raise EnvironmentPending(
"Cannot connect to environment using %s "
"(perhaps still initializing): %s" % (
chosen.dns_name, str(e)))
yield self.wait_for_initialization(client)
returnValue(client)
def setUp(self):
self.sshclient = SSHClient()
self.log = self.capture_logging("juju.state.sshforward")
class SSHClientTest(TestCase):
def setUp(self):
self.sshclient = SSHClient()
self.log = self.capture_logging("juju.state.sshforward")
def test_is_zookeeper_client(self):
self.assertTrue(isinstance(self.sshclient, ZookeeperClient))
def test_close_while_not_connected_does_nothing(self):
# Hah! Nothing! But it doesn't blow up either.
self.sshclient.close()
def test_internal_connect_behavior(self):
"""Verify the order of operations for sshclient._internal_connect."""
zkconnect = self.mocker.replace(ZookeeperClient.connect)
zkclose = self.mocker.replace(ZookeeperClient.close)
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
process = self.mocker.mock(Process)
with self.mocker.order():
# First, get the forwarding going, targetting the remote
# address provided.
forward("ubuntu",
MATCH_PORT,
"remote",
2181,
process_protocol=MATCH_PROTOCOL,
share=False)
self.mocker.result(process)
# Next ensure the port check succeeds
thread(MATCH_FUNC)
self.mocker.result(succeed(True))
# Then, connect to localhost, though the set up proxy.
zkconnect(MATCH_LOCALHOST_PORT, MATCH_TIMEOUT)
# Just a marker to ensure the following happens as a
# side effect of actually closing the SSHClient.
process.pre_close_marker
zkclose()
process.signalProcess("TERM")
process.loseConnection()
# There we go!
self.mocker.replay()
self.sshclient.connect("remote:2181", timeout=123)
# Trick to ensure process.close() didn't happen
# before this point. This only works because we're
# asking mocker to order events here.
process.pre_close_marker
self.sshclient.close()
def test_new_timeout_after_port_probe(self):
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
original_time = time.time
times = [220, 200, 200]
def get_time():
if times:
return times.pop()
return original_time()
self.patch(time, "time", get_time)
protocol = self.mocker.mock()
forward("ubuntu",
MATCH_PORT,
"remote",
2181,
process_protocol=MATCH_PROTOCOL,
share=False)
self.mocker.result(protocol)
thread(MATCH_FUNC)
self.mocker.result(succeed(True))
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
d = self.sshclient.connect("remote:2181", timeout=20)
self.failUnlessFailure(d, ConnectionTimeoutException)
return d
def test_tunnel_port_open_error(self):
"""Errors when probing the port are reported on the connect deferred.
Port probing errors are converted to connectiontimeout exceptions.
"""
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
protocol = self.mocker.mock()
forward("ubuntu",
MATCH_PORT,
"remote",
2181,
process_protocol=MATCH_PROTOCOL,
share=False)
self.mocker.result(protocol)
thread(MATCH_FUNC)
self.mocker.result(fail(socket.error("a", )))
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
d = self.sshclient.connect("remote:2181", timeout=20)
self.failUnlessFailure(d, ConnectionTimeoutException)
return d
def test_tunnel_client_error(self):
"""A zkclient connect error is reported on the sshclient deferred.
Client connection errors are propogated as is.
"""
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
protocol = self.mocker.mock()
forward("ubuntu",
MATCH_PORT,
"remote",
2181,
process_protocol=MATCH_PROTOCOL,
share=False)
self.mocker.result(protocol)
thread(MATCH_FUNC)
def wait_result(func):
return succeed(True)
self.mocker.call(wait_result)
zkconnect = self.mocker.replace(ZookeeperClient.connect)
zkconnect(MATCH_LOCALHOST_PORT, MATCH_TIMEOUT)
self.mocker.result(fail(OSError()))
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
d = self.sshclient.connect("remote:2181", timeout=20)
self.failUnlessFailure(d, OSError)
return d
def test_share_connection(self):
"""Connection sharing requests are passed to port_forward().
"""
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
protocol = self.mocker.mock()
forward("ubuntu",
MATCH_PORT,
"remote",
2181,
process_protocol=MATCH_PROTOCOL,
share=True)
self.mocker.result(protocol)
thread(MATCH_FUNC)
def wait_result(func):
return succeed(True)
self.mocker.call(wait_result)
zkconnect = self.mocker.replace(ZookeeperClient.connect)
zkconnect(MATCH_LOCALHOST_PORT, MATCH_TIMEOUT)
self.mocker.result(True)
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
yield self.sshclient.connect("remote:2181", timeout=20, share=True)
@inlineCallbacks
def test_connect(self):
"""Test normal connection w/o retry loop."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect("remote:2181",
timeout=MATCH_TIMEOUT,
share=False)
self.mocker.result(succeed(True))
self.mocker.replay()
yield self.sshclient.connect("remote:2181", timeout=123)
@inlineCallbacks
def test_connect_no_connection(self):
"""Test sequence of NoConnection failures, followed by success."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(
TestDeferredSequence(
[NoConnection(), NoConnection(), True],
"remote:2181",
timeout=MATCH_TIMEOUT,
share=False))
self.mocker.count(3, 3)
self.mocker.replay()
yield self.sshclient.connect("remote:2181", timeout=123)
@inlineCallbacks
def test_connect_invalid_host(self):
"""Test connect to invalid host will raise exception asap."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(
TestDeferredSequence(
[NoConnection(), InvalidHost(),
succeed(True)],
"remote:2181",
timeout=MATCH_TIMEOUT,
share=False))
self.mocker.count(2, 2)
self.mocker.replay()
yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123), InvalidHost)
@inlineCallbacks
def test_connect_invalid_user(self):
"""Test connect with invalid user will raise exception asap."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(
TestDeferredSequence(
[NoConnection(), InvalidUser(),
succeed(True)],
"remote:2181",
timeout=MATCH_TIMEOUT,
share=False))
self.mocker.count(2, 2)
self.mocker.replay()
yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123), InvalidUser)
@inlineCallbacks
def test_connect_timeout(self):
"""Test that retry fails after timeout in retry loop."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(
TestDeferredSequence(
[NoConnection(), NoConnection(), True],
"remote:2181",
timeout=MATCH_TIMEOUT,
share=False))
self.mocker.count(2, 2)
original_time = time.time
times = [220, 215, 210, 205, 200]
def get_time():
if times:
return times.pop()
return original_time()
self.patch(time, "time", get_time)
self.mocker.replay()
ex = yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123),
ConnectionTimeoutException)
self.assertEqual(str(ex),
"could not connect before timeout after 2 retries")
@inlineCallbacks
def test_connect_tunnel_portwatcher_timeout(self):
"""Test that retry fails after timeout seen in tunnel portwatcher."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(
TestDeferredSequence([
NoConnection(),
NoConnection(),
ConnectionTimeoutException(), True
],
"remote:2181",
timeout=MATCH_TIMEOUT,
share=False))
self.mocker.count(3, 3)
self.mocker.replay()
ex = yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123),
ConnectionTimeoutException)
self.assertEqual(str(ex),
"could not connect before timeout after 3 retries")
class ConnectionTest(TestCase):
def setUp(self):
super(ConnectionTest, self).setUp()
self.username = pwd.getpwuid(os.getuid())[0]
self.log = self.capture_logging("juju.state.sshforward")
self.old_user_name = SSHClient.remote_user
SSHClient.remote_user = self.username
self.client = SSHClient()
zookeeper.set_debug_level(0)
def tearDown(self):
super(ConnectionTest, self).tearDown()
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_DEBUG)
SSHClient.remote_user = self.old_user_name
def test_connect(self):
"""
Forwarding a port spawns an ssh process with port forwarding arguments.
"""
connect_deferred = self.client.connect(
get_test_zookeeper_address(), timeout=20)
def validate_connected(client):
self.assertTrue(client.connected)
client.close()
connect_deferred.addCallback(validate_connected)
return connect_deferred
def test_invalid_host(self):
"""
if a connection can not be made before a timeout period, an exception
is raised.
with the sshclient layer, tihs test no longer returns a failure..
and its hard to cleanup the process tunnel..
"""
SSHClient.remote_user = "******"
connect_deferred = self.client.connect(
"foobar.example.com:2181", timeout=10)
self.failUnlessFailure(connect_deferred, NoConnection)
def validate_log(result):
output = self.log.getvalue()
self.assertTrue(output.strip().startswith(
"Invalid host for SSH forwarding"))
connect_deferred.addCallback(validate_log)
return connect_deferred
def test_invalid_user(self):
"""
if a connection can not be made before a timeout period, an exception
is raised.
with the sshclient layer, tihs test no longer returns a failure..
and its hard to cleanup the process tunnel..
"""
SSHClient.remote_user = "******"
connect_deferred = self.client.connect(
get_test_zookeeper_address(), timeout=10)
self.failUnlessFailure(connect_deferred, NoConnection)
def validate_log(result):
output = self.log.getvalue()
self.assertEqual(output.strip(), "Invalid SSH key")
connect_deferred.addCallback(validate_log)
return connect_deferred
class ConnectionTest(TestCase):
def setUp(self):
super(ConnectionTest, self).setUp()
self.username = pwd.getpwuid(os.getuid())[0]
self.log = self.capture_logging("juju.state.sshforward")
self.old_user_name = SSHClient.remote_user
SSHClient.remote_user = self.username
self.client = SSHClient()
zookeeper.set_debug_level(0)
def tearDown(self):
super(ConnectionTest, self).tearDown()
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_DEBUG)
SSHClient.remote_user = self.old_user_name
def test_connect(self):
"""
Forwarding a port spawns an ssh process with port forwarding arguments.
"""
connect_deferred = self.client.connect(get_test_zookeeper_address(),
timeout=20)
def validate_connected(client):
self.assertTrue(client.connected)
client.close()
connect_deferred.addCallback(validate_connected)
return connect_deferred
def test_invalid_host(self):
"""
if a connection can not be made before a timeout period, an exception
is raised.
with the sshclient layer, tihs test no longer returns a failure..
and its hard to cleanup the process tunnel..
"""
SSHClient.remote_user = "******"
connect_deferred = self.client.connect("foobar.example.com:2181",
timeout=10)
self.failUnlessFailure(connect_deferred, NoConnection)
def validate_log(result):
output = self.log.getvalue()
self.assertTrue(
output.strip().startswith("Invalid host for SSH forwarding"))
connect_deferred.addCallback(validate_log)
return connect_deferred
def test_invalid_user(self):
"""
if a connection can not be made before a timeout period, an exception
is raised.
with the sshclient layer, tihs test no longer returns a failure..
and its hard to cleanup the process tunnel..
"""
SSHClient.remote_user = "******"
connect_deferred = self.client.connect(get_test_zookeeper_address(),
timeout=10)
self.failUnlessFailure(connect_deferred, NoConnection)
def validate_log(result):
output = self.log.getvalue()
self.assertEqual(output.strip(), "Invalid SSH key")
connect_deferred.addCallback(validate_log)
return connect_deferred
def setUp(self):
self.sshclient = SSHClient()
self.log = self.capture_logging("juju.state.sshforward")
class SSHClientTest(TestCase):
def setUp(self):
self.sshclient = SSHClient()
self.log = self.capture_logging("juju.state.sshforward")
def test_is_zookeeper_client(self):
self.assertTrue(isinstance(self.sshclient, ZookeeperClient))
def test_close_while_not_connected_does_nothing(self):
# Hah! Nothing! But it doesn't blow up either.
self.sshclient.close()
def test_internal_connect_behavior(self):
"""Verify the order of operations for sshclient._internal_connect."""
zkconnect = self.mocker.replace(ZookeeperClient.connect)
zkclose = self.mocker.replace(ZookeeperClient.close)
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
process = self.mocker.mock(Process)
with self.mocker.order():
# First, get the forwarding going, targetting the remote
# address provided.
forward("ubuntu", MATCH_PORT, "remote", 2181,
process_protocol=MATCH_PROTOCOL, share=False)
self.mocker.result(process)
# Next ensure the port check succeeds
thread(MATCH_FUNC)
self.mocker.result(succeed(True))
# Then, connect to localhost, though the set up proxy.
zkconnect(MATCH_LOCALHOST_PORT, MATCH_TIMEOUT)
# Just a marker to ensure the following happens as a
# side effect of actually closing the SSHClient.
process.pre_close_marker
zkclose()
process.signalProcess("TERM")
process.loseConnection()
# There we go!
self.mocker.replay()
self.sshclient.connect(
"remote:2181", timeout=123)
# Trick to ensure process.close() didn't happen
# before this point. This only works because we're
# asking mocker to order events here.
process.pre_close_marker
self.sshclient.close()
def test_new_timeout_after_port_probe(self):
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
original_time = time.time
times = [220, 200, 200]
def get_time():
if times:
return times.pop()
return original_time()
self.patch(time, "time", get_time)
protocol = self.mocker.mock()
forward("ubuntu", MATCH_PORT, "remote", 2181,
process_protocol=MATCH_PROTOCOL, share=False)
self.mocker.result(protocol)
thread(MATCH_FUNC)
self.mocker.result(succeed(True))
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
d = self.sshclient.connect("remote:2181", timeout=20)
self.failUnlessFailure(d, ConnectionTimeoutException)
return d
def test_tunnel_port_open_error(self):
"""Errors when probing the port are reported on the connect deferred.
Port probing errors are converted to connectiontimeout exceptions.
"""
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
protocol = self.mocker.mock()
forward("ubuntu", MATCH_PORT, "remote", 2181,
process_protocol=MATCH_PROTOCOL, share=False)
self.mocker.result(protocol)
thread(MATCH_FUNC)
self.mocker.result(fail(socket.error("a",)))
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
d = self.sshclient.connect("remote:2181", timeout=20)
self.failUnlessFailure(d, ConnectionTimeoutException)
return d
def test_tunnel_client_error(self):
"""A zkclient connect error is reported on the sshclient deferred.
Client connection errors are propogated as is.
"""
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
protocol = self.mocker.mock()
forward("ubuntu", MATCH_PORT, "remote", 2181,
process_protocol=MATCH_PROTOCOL, share=False)
self.mocker.result(protocol)
thread(MATCH_FUNC)
def wait_result(func):
return succeed(True)
self.mocker.call(wait_result)
zkconnect = self.mocker.replace(ZookeeperClient.connect)
zkconnect(MATCH_LOCALHOST_PORT, MATCH_TIMEOUT)
self.mocker.result(fail(OSError()))
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
d = self.sshclient.connect(
"remote:2181", timeout=20)
self.failUnlessFailure(d, OSError)
return d
def test_share_connection(self):
"""Connection sharing requests are passed to port_forward().
"""
forward = self.mocker.replace("juju.state.sshforward.forward_port")
thread = self.mocker.replace("twisted.internet.threads.deferToThread")
protocol = self.mocker.mock()
forward("ubuntu", MATCH_PORT, "remote", 2181,
process_protocol=MATCH_PROTOCOL, share=True)
self.mocker.result(protocol)
thread(MATCH_FUNC)
def wait_result(func):
return succeed(True)
self.mocker.call(wait_result)
zkconnect = self.mocker.replace(ZookeeperClient.connect)
zkconnect(MATCH_LOCALHOST_PORT, MATCH_TIMEOUT)
self.mocker.result(True)
protocol.signalProcess("TERM")
protocol.loseConnection()
self.mocker.replay()
yield self.sshclient.connect("remote:2181", timeout=20, share=True)
@inlineCallbacks
def test_connect(self):
"""Test normal connection w/o retry loop."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect(
"remote:2181", timeout=MATCH_TIMEOUT, share=False)
self.mocker.result(succeed(True))
self.mocker.replay()
yield self.sshclient.connect("remote:2181", timeout=123)
@inlineCallbacks
def test_connect_no_connection(self):
"""Test sequence of NoConnection failures, followed by success."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(TestDeferredSequence(
[NoConnection(), NoConnection(), True],
"remote:2181", timeout=MATCH_TIMEOUT, share=False))
self.mocker.count(3, 3)
self.mocker.replay()
yield self.sshclient.connect("remote:2181", timeout=123)
@inlineCallbacks
def test_connect_invalid_host(self):
"""Test connect to invalid host will raise exception asap."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(TestDeferredSequence(
[NoConnection(), InvalidHost(), succeed(True)],
"remote:2181", timeout=MATCH_TIMEOUT, share=False))
self.mocker.count(2, 2)
self.mocker.replay()
yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123),
InvalidHost)
@inlineCallbacks
def test_connect_invalid_user(self):
"""Test connect with invalid user will raise exception asap."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(TestDeferredSequence(
[NoConnection(), InvalidUser(), succeed(True)],
"remote:2181", timeout=MATCH_TIMEOUT, share=False))
self.mocker.count(2, 2)
self.mocker.replay()
yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123),
InvalidUser)
@inlineCallbacks
def test_connect_timeout(self):
"""Test that retry fails after timeout in retry loop."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(TestDeferredSequence(
[NoConnection(), NoConnection(), True],
"remote:2181", timeout=MATCH_TIMEOUT, share=False))
self.mocker.count(2, 2)
original_time = time.time
times = [220, 215, 210, 205, 200]
def get_time():
if times:
return times.pop()
return original_time()
self.patch(time, "time", get_time)
self.mocker.replay()
ex = yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123),
ConnectionTimeoutException)
self.assertEqual(
str(ex),
"could not connect before timeout after 2 retries")
@inlineCallbacks
def test_connect_tunnel_portwatcher_timeout(self):
"""Test that retry fails after timeout seen in tunnel portwatcher."""
mock_client = self.mocker.patch(self.sshclient)
mock_client._internal_connect
self.mocker.call(TestDeferredSequence(
[NoConnection(), NoConnection(),
ConnectionTimeoutException(), True],
"remote:2181", timeout=MATCH_TIMEOUT, share=False))
self.mocker.count(3, 3)
self.mocker.replay()
ex = yield self.assertFailure(
self.sshclient.connect("remote:2181", timeout=123),
ConnectionTimeoutException)
self.assertEqual(
str(ex),
"could not connect before timeout after 3 retries")
