def test_wait_for_initialize(self):
"""
A connection to a zookeeper that is running, but whose juju state
is not ready, should wait until that state is ready.
"""
client = ZookeeperClient()
self.client = client # for poke_zk
self.mock_connect(False, succeed(client))
self.mocker.replay()
zookeeper.set_debug_level(0)
yield client.connect(get_test_zookeeper_address())
provider = DummyProvider(ProviderMachine("i-amok", "foo.example.com"))
d = provider.connect()
client_result = []
d.addCallback(client_result.append)
# Give it a chance to do it incorrectly.
yield self.poke_zk()
try:
self.assertEquals(client_result, [])
yield client.create("/initialized")
yield d
self.assertTrue(client_result, client_result)
self.assertIdentical(client_result[0], client)
finally:
deleteTree("/", client.handle)
client.close()
def setUp(self):
self.log = self.capture_logging("juju.state.init")
zookeeper.set_debug_level(0)
self.client = ZookeeperClient(get_test_zookeeper_address())
self.identity = make_identity("admin:genie")
self.layout = StateHierarchy(self.client, self.identity, "i-abcdef",
"dummy")
return self.client.connect()
def tearDown(self):
# Close and reopen connection, so that watches set during
# testing are not affected by the cleaning up.
self.client.close()
client = ZookeeperClient(get_test_zookeeper_address())
yield client.connect()
deleteTree(handle=client.handle)
client.close()
yield super(StateTestBase, self).tearDown()
def connect(self, share=False):
"""Connect to juju's zookeeper.
"""
state = yield self.load_state()
if not state:
raise EnvironmentNotFound()
client = yield ZookeeperClient(state["zookeeper-address"]).connect()
yield ZookeeperConnect(self).wait_for_initialization(client)
returnValue(client)
def test_watch_stops_on_closed_connection(self):
"""Verify watches stops when the connection is closed."""
# Use a separate client connection for watching so it can be
# disconnected.
watch_client = ZookeeperClient(get_test_zookeeper_address())
yield watch_client.connect()
watch_base = StateBase(watch_client)
wait_callback = Deferred()
finish_callback = Deferred()
calls = []
def watcher(old_topology, new_topology):
calls.append((old_topology, new_topology))
wait_callback.callback(True)
return finish_callback
# Start watching.
yield watch_base._watch_topology(watcher)
# Create the topology.
topology = InternalTopology()
topology.add_machine("m-0")
yield self.set_topology(topology)
# Hold off until callback is started.
yield wait_callback
# Change the topology.
topology.add_machine("m-1")
yield self.set_topology(topology)
# Ensure that the watch has been called just once so far
# (although still pending due to the finish_callback).
self.assertEquals(len(calls), 1)
# Now disconnect the client.
watch_client.close()
self.assertFalse(watch_client.connected)
self.assertTrue(self.client.connected)
# Change the topology again.
topology.add_machine("m-2")
yield self.set_topology(topology)
# Allow the first call to be completed, starting a process of
# watching for the next change. At this point, the watch will
# encounter that the client is disconnected.
finish_callback.callback(True)
# Give a chance for something bad to happen.
yield self.poke_zk()
# Ensure the watch was still not called.
self.assertEquals(len(calls), 1)
def connect(self, share=False):
"""Connect to the zookeeper juju running in the machine provider.
@param share: Requests sharing of the connection with other clients
attempting to connect to the same provider, if that's feasible.
Unused for the dummy provider.
"""
return ZookeeperClient(os.environ.get("ZOOKEEPER_ADDRESS",
"127.0.0.1:2181"),
session_timeout=1000).connect()
def open_client(self, credentials=None):
"""
Open a zookeeper client, optionally authenticating with the
credentials if given.
"""
client = ZookeeperClient("127.0.0.1:2181")
self.clients.append(client)
yield client.connect()
if credentials:
d = client.add_auth("digest", credentials)
# hack to keep auth fast
yield client.exists("/")
yield d
returnValue(client)
def connect(self):
"""Return an authenticated connection to the juju zookeeper."""
hosts = self.config["zookeeper_servers"]
self.client = yield ZookeeperClient().connect(hosts)
principals = self.config.get("principals", ())
for principal in principals:
self.client.add_auth("digest", principal)
# bug work around to keep auth fast
if principals:
yield self.client.exists("/")
returnValue(self.client)
def setUp(self):
super(CharmPublisherTest, self).setUp()
zookeeper.set_debug_level(0)
self.charm = CharmDirectory(self.sample_dir1)
self.charm_id = local_charm_id(self.charm)
self.charm_key = under.quote(self.charm_id)
# provider storage key
self.charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, self.charm.get_sha256()))
self.client = ZookeeperClient(get_test_zookeeper_address())
self.storage_dir = self.makeDir()
self.storage = FileStorage(self.storage_dir)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.client.connect()
yield self.client.create("/charms")
def test_watch_stops_on_early_closed_connection(self):
"""Verify watches stops when the connection is closed early.
_watch_topology chains from an exists_and_watch to a
get_and_watch. This test ensures that this chaining will fail
gracefully if the connection is closed before this chaining
can occur.
"""
# Use a separate client connection for watching so it can be
# disconnected.
watch_client = ZookeeperClient(get_test_zookeeper_address())
yield watch_client.connect()
watch_base = StateBase(watch_client)
calls = []
@inlineCallbacks
def watcher(old_topology, new_topology):
calls.append((old_topology, new_topology))
# Create the topology.
topology = InternalTopology()
topology.add_machine("m-0")
yield self.set_topology(topology)
# Now disconnect the client.
watch_client.close()
self.assertFalse(watch_client.connected)
self.assertTrue(self.client.connected)
# Start watching.
yield watch_base._watch_topology(watcher)
# Change the topology, this will trigger the watch.
topology.add_machine("m-1")
yield self.set_topology(topology)
# Give a chance for something bad to happen.
yield self.poke_zk()
# Ensure the watcher was never called, because its client was
# disconnected.
self.assertEquals(len(calls), 0)
def test_managed_zookeeper(self):
zookeeper.set_debug_level(0)
# Start zookeeper
data_dir = self.makeDir()
instance = Zookeeper(data_dir, 12345)
yield instance.start()
self.assertTrue(instance.running)
# Connect a client
client = ZookeeperClient("127.0.1.1:12345")
yield client.connect()
stat = yield client.exists("/")
yield client.close()
self.assertTrue(stat)
# Stop Instance
yield instance.stop()
self.assertFalse(instance.running)
self.assertFalse(os.path.exists(data_dir))
def test_fast_connection(self):
"""Verify connection when requirements are available at time of conn.
This includes /initialized is already set. In addition, also
verifies that if multiple ZKs are available, one is selected
via random.choice.
"""
log = self.capture_logging(level=logging.DEBUG)
client = ZookeeperClient()
self.mock_connect(False, succeed(client))
def get_choice(lst):
for item in lst:
if item.dns_name == "foo.example.com":
return item
raise AssertionError("expected choice not seen")
self.patch(random, "choice", get_choice)
self.mocker.replay()
provider = DummyProvider(
ProviderMachine("i-am-picked", "foo.example.com"),
ProviderMachine("i-was-not", "bar.example.com"))
zookeeper.set_debug_level(0)
yield client.connect(get_test_zookeeper_address())
try:
yield client.create("/initialized")
yield provider.connect()
self.assertEqual(
"Connecting to environment...\n"
"Connecting to environment using foo.example.com...\n"
"Environment is initialized.\n"
"Connected to environment.\n", log.getvalue())
finally:
deleteTree("/", client.handle)
client.close()
def bootstrap(self):
"""Bootstrap a local development environment.
"""
# Check for existing environment
state = yield self.load_state()
if state is not False:
raise ProviderError("Environment already bootstrapped")
# Check for required packages
log.info("Checking for required packages...")
missing = check_packages(*REQUIRED_PACKAGES)
if missing:
raise ProviderError("Missing packages %s" % (
", ".join(sorted(list(missing)))))
# Get/create directory for zookeeper and files
zookeeper_dir = os.path.join(self._directory, "zookeeper")
if not os.path.exists(zookeeper_dir):
os.makedirs(zookeeper_dir)
# Start networking, and get an open port.
log.info("Starting networking...")
net = Network("default", subnet=122)
# Start is a noop if its already started, which it is by default,
# per libvirt-bin package installation
yield net.start()
net_attributes = yield net.get_attributes()
port = get_open_port(net_attributes["ip"]["address"])
# Start zookeeper
log.info("Starting zookeeper...")
# Run zookeeper as the current user, unless we're being run as root
# in which case run zookeeper as the 'zookeeper' user.
zookeeper_user = None
if os.geteuid() == 0:
zookeeper_user = "******"
zookeeper = Zookeeper(zookeeper_dir,
port=port,
host=net_attributes["ip"]["address"],
user=zookeeper_user, group=zookeeper_user)
yield zookeeper.start()
# Starting provider storage server
log.info("Starting storage server...")
storage_server = StorageServer(
pid_file=os.path.join(self._directory, "storage-server.pid"),
storage_dir=os.path.join(self._directory, "files"),
host=net_attributes["ip"]["address"],
port=get_open_port(net_attributes["ip"]["address"]),
log_file=os.path.join(self._directory, "storage-server.log"))
yield storage_server.start()
# Save the zookeeper start to provider storage.
yield self.save_state({"zookeeper-instances": ["local"],
"zookeeper-address": zookeeper.address})
# Initialize the zookeeper state
log.debug("Initializing state...")
admin_identity = make_identity(
"admin:%s" % self.config["admin-secret"])
client = ZookeeperClient(zookeeper.address)
yield client.connect()
hierarchy = StateHierarchy(client, admin_identity, "local", "local")
yield hierarchy.initialize()
# Store user credentials from the running user
try:
public_key = get_user_authorized_keys(self.config)
public_key = public_key.strip()
except LookupError, e:
raise ProviderError(str(e))
from binascii import crc32
from functools import wraps
from itertools import count
import json
import struct
import sys
from twisted.logger import Logger, globalLogBeginner, textFileLogObserver
from twisted.internet.defer import Deferred, inlineCallbacks, maybeDeferred
from twisted.internet.task import react
from twisted.internet.protocol import Protocol, Factory
from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol
from txzookeeper import ZookeeperClient
zk = ZookeeperClient(servers='localhost:2181')
from parsley import makeProtocol
from characteristic import attributes, Attribute
log = Logger()
def encodeString(string):
assert isinstance(string, bytes)
return struct.pack('>h', len(string)) + string
def encodeBytes(string):
assert isinstance(string, bytes)
return struct.pack('>i', len(string)) + string
def setUp(self):
zookeeper.set_debug_level(0)
self.client = ZookeeperClient(get_test_zookeeper_address())
yield self.client.connect()
self.path = "/zoo"
def setUp(self):
yield super(RemoveTreeTest, self).setUp()
zookeeper.set_debug_level(0)
self.client = ZookeeperClient(get_test_zookeeper_address())
yield self.client.connect()
def get_zookeeper_client(self):
client = ZookeeperClient(get_test_zookeeper_address(),
session_timeout=1000)
return client
