def _provision_scripts(zookeeper_hosts):
service = UpstartService("juju-provision-agent")
service.set_description("Juju provisioning agent")
service.set_environ({"JUJU_ZOOKEEPER": zookeeper_hosts})
service.set_command(
"python -m juju.agents.provision --nodaemon "
"--logfile /var/log/juju/provision-agent.log "
"--session-file /var/run/juju/provision-agent.zksession")
return service.get_cloud_init_commands()
def _machine_scripts(machine_id, zookeeper_hosts):
service = UpstartService("juju-machine-agent")
service.set_description("Juju machine agent")
service.set_environ(
{"JUJU_MACHINE_ID": machine_id, "JUJU_ZOOKEEPER": zookeeper_hosts})
service.set_command(
"python -m juju.agents.machine --nodaemon "
"--logfile /var/log/juju/machine-agent.log "
"--session-file /var/run/juju/machine-agent.zksession")
return service.get_cloud_init_commands()
class ManagedMachineAgent(object):
agent_module = "juju.agents.machine"
def __init__(
self, juju_unit_namespace, zookeeper_hosts=None,
machine_id="0", log_file=None, juju_directory="/var/lib/juju",
public_key=None, juju_origin="ppa", juju_series=None):
"""
:param juju_series: The release series to use (maverick, natty, etc).
:param machine_id: machine id for the local machine.
:param zookeeper_hosts: Zookeeper hosts to connect.
:param log_file: A file to use for the agent logs.
:param juju_directory: The directory to use for all state and logs.
:param juju_unit_namespace: The machine agent will create units with
a known a prefix to allow for multiple users and multiple
environments to create containers. The namespace should be
unique per user and per environment.
:param public_key: An SSH public key (string) that will be
used in the container for access.
"""
self._juju_origin = juju_origin
if self._juju_origin is None:
origin, source = get_default_origin()
if origin == BRANCH:
origin = source
self._juju_origin = origin
env = {"JUJU_MACHINE_ID": machine_id,
"JUJU_ZOOKEEPER": zookeeper_hosts,
"JUJU_HOME": juju_directory,
"JUJU_ORIGIN": self._juju_origin,
"JUJU_UNIT_NS": juju_unit_namespace,
"JUJU_SERIES": juju_series,
"PYTHONPATH": ":".join(sys.path)}
if public_key:
env["JUJU_PUBLIC_KEY"] = public_key
self._service = UpstartService(
"juju-%s-machine-agent" % juju_unit_namespace, use_sudo=True)
self._service.set_description(
"Juju machine agent for %s" % juju_unit_namespace)
self._service.set_environ(env)
self._service_args = [
"/usr/bin/python", "-m", self.agent_module,
"--nodaemon", "--logfile", log_file,
"--session-file",
"/var/run/juju/%s-machine-agent.zksession" % juju_unit_namespace]
@property
def juju_origin(self):
return self._juju_origin
def start(self):
"""Start the machine agent."""
self._service.set_command(" ".join(self._service_args))
return self._service.start()
def stop(self):
"""Stop the machine agent."""
return self._service.destroy()
def is_running(self):
"""Boolean value, true if the machine agent is running."""
return self._service.is_running()
class UpstartServiceTest(TestCase):
@inlineCallbacks
def setUp(self):
yield super(UpstartServiceTest, self).setUp()
self.init_dir = self.makeDir()
self.conf = os.path.join(self.init_dir, "some-name.conf")
self.output = "/tmp/some-name.output"
self.patch(UpstartService, "init_dir", self.init_dir)
self.service = UpstartService("some-name")
def setup_service(self):
self.service.set_description("a wretched hive of scum and villainy")
self.service.set_command("/bin/imagination-failure --no-ideas")
self.service.set_environ({"LIGHTSABER": "civilised weapon"})
def setup_mock(self):
self.check_call = self.mocker.replace("subprocess.check_call")
self.getProcessOutput = self.mocker.replace(
"twisted.internet.utils.getProcessOutput")
def mock_status(self, result):
self.getProcessOutput("/sbin/status", ["some-name"])
self.mocker.result(result)
def mock_call(self, args, output=None):
self.check_call(args, KWARGS)
if output is None:
self.mocker.result(0)
else:
def write(ANY, **_):
with open(self.output, "w") as f:
f.write(output)
self.mocker.call(write)
def mock_start(self, output=None):
self.mock_call(("/sbin/start", "some-name"), output)
def mock_stop(self):
self.mock_call(("/sbin/stop", "some-name"))
def mock_check_success(self):
for _ in range(5):
self.mock_status(succeed("blah start/running blah 12345"))
def mock_check_unstable(self):
for _ in range(4):
self.mock_status(succeed("blah start/running blah 12345"))
self.mock_status(succeed("blah start/running blah 12346"))
def mock_check_not_running(self):
self.mock_status(succeed("blah"))
def write_dummy_conf(self):
with open(self.conf, "w") as f:
f.write("dummy")
def assert_dummy_conf(self):
with open(self.conf) as f:
self.assertEquals(f.read(), "dummy")
def assert_no_conf(self):
self.assertFalse(os.path.exists(self.conf))
def assert_conf(self, name="test_standard_install"):
with open(os.path.join(DATA_DIR, name)) as expected:
with open(self.conf) as actual:
self.assertEquals(actual.read(), expected.read())
def test_is_installed(self):
"""Check is_installed depends on conf file existence"""
self.assertFalse(self.service.is_installed())
self.write_dummy_conf()
self.assertTrue(self.service.is_installed())
def test_init_dir(self):
"""
Check is_installed still works when init_dir specified explicitly
"""
self.patch(UpstartService, "init_dir", "/BAD/PATH")
self.service = UpstartService("some-name", init_dir=self.init_dir)
self.setup_service()
self.assertFalse(self.service.is_installed())
self.write_dummy_conf()
self.assertTrue(self.service.is_installed())
@inlineCallbacks
def test_is_running(self):
"""
Check is_running interprets status output (when service is installed)
"""
self.setup_mock()
self.mock_status(succeed("blah stop/waiting blah"))
self.mock_status(succeed("blah blob/gibbering blah"))
self.mock_status(succeed("blah start/running blah 12345"))
self.mocker.replay()
# Won't hit status; conf is not installed
self.assertFalse((yield self.service.is_running()))
self.write_dummy_conf()
# These 3 calls correspond to the first 3 mock_status calls above
self.assertFalse((yield self.service.is_running()))
self.assertFalse((yield self.service.is_running()))
self.assertTrue((yield self.service.is_running()))
@inlineCallbacks
def test_is_stable_yes(self):
self.setup_mock()
self.mock_check_success()
self.mocker.replay()
self.write_dummy_conf()
self.assertTrue((yield self.service.is_stable()))
@inlineCallbacks
def test_is_stable_no(self):
self.setup_mock()
self.mock_check_unstable()
self.mocker.replay()
self.write_dummy_conf()
self.assertFalse((yield self.service.is_stable()))
@inlineCallbacks
def test_is_stable_not_running(self):
self.setup_mock()
self.mock_check_not_running()
self.mocker.replay()
self.write_dummy_conf()
self.assertFalse((yield self.service.is_stable()))
@inlineCallbacks
def test_is_stable_not_even_installed(self):
self.assertFalse((yield self.service.is_stable()))
@inlineCallbacks
def test_get_pid(self):
"""
Check get_pid interprets status output (when service is installed)
"""
self.setup_mock()
self.mock_status(succeed("blah stop/waiting blah"))
self.mock_status(succeed("blah blob/gibbering blah"))
self.mock_status(succeed("blah start/running blah 12345"))
self.mocker.replay()
# Won't hit status; conf is not installed
self.assertEquals((yield self.service.get_pid()), None)
self.write_dummy_conf()
# These 3 calls correspond to the first 3 mock_status calls above
self.assertEquals((yield self.service.get_pid()), None)
self.assertEquals((yield self.service.get_pid()), None)
self.assertEquals((yield self.service.get_pid()), 12345)
@inlineCallbacks
def test_basic_install(self):
"""Check a simple UpstartService writes expected conf file"""
e = yield self.assertFailure(self.service.install(), ServiceError)
self.assertEquals(str(e), "Cannot render .conf: no description set")
self.service.set_description("uninteresting service")
e = yield self.assertFailure(self.service.install(), ServiceError)
self.assertEquals(str(e), "Cannot render .conf: no command set")
self.service.set_command("/bin/false")
yield self.service.install()
self.assert_conf("test_basic_install")
@inlineCallbacks
def test_less_basic_install(self):
"""Check conf for a different UpstartService (which sets an env var)"""
self.service.set_description("pew pew pew blam")
self.service.set_command("/bin/deathstar --ignore-ewoks endor")
self.service.set_environ({"FOO": "bar baz qux", "PEW": "pew"})
self.service.set_output_path("/somewhere/else")
yield self.service.install()
self.assert_conf("test_less_basic_install")
def test_install_via_script(self):
"""Check that the output-as-script form does the right thing"""
self.setup_service()
install, start = self.service.get_cloud_init_commands()
os.system(install)
self.assert_conf()
self.assertEquals(start, "/sbin/start some-name")
@inlineCallbacks
def test_start_not_installed(self):
"""Check that .start() also installs if necessary"""
self.setup_mock()
self.mock_status(succeed("blah stop/waiting blah"))
self.mock_start()
self.mock_check_success()
self.mocker.replay()
self.setup_service()
yield self.service.start()
self.assert_conf()
@inlineCallbacks
def test_start_not_started_stable(self):
"""Check that .start() starts if stopped, and checks for stable pid"""
self.write_dummy_conf()
self.setup_mock()
self.mock_status(succeed("blah stop/waiting blah"))
self.mock_start("ignored")
self.mock_check_success()
self.mocker.replay()
self.setup_service()
yield self.service.start()
self.assert_dummy_conf()
@inlineCallbacks
def test_start_not_started_unstable(self):
"""Check that .start() starts if stopped, and raises on unstable pid"""
self.write_dummy_conf()
self.setup_mock()
self.mock_status(succeed("blah stop/waiting blah"))
self.mock_start("kangaroo")
self.mock_check_unstable()
self.mocker.replay()
self.setup_service()
e = yield self.assertFailure(self.service.start(), ServiceError)
self.assertEquals(
str(e), "Failed to start job some-name; got output:\nkangaroo")
self.assert_dummy_conf()
@inlineCallbacks
def test_start_not_started_failure(self):
"""Check that .start() starts if stopped, and raises on no pid"""
self.write_dummy_conf()
self.setup_mock()
self.mock_status(succeed("blah stop/waiting blah"))
self.mock_start()
self.mock_check_not_running()
self.mocker.replay()
self.setup_service()
e = yield self.assertFailure(self.service.start(), ServiceError)
self.assertEquals(
str(e), "Failed to start job some-name; no output detected")
self.assert_dummy_conf()
@inlineCallbacks
def test_start_started(self):
"""Check that .start() does nothing if already running"""
self.write_dummy_conf()
self.setup_mock()
self.mock_status(succeed("blah start/running blah 12345"))
self.mocker.replay()
self.setup_service()
yield self.service.start()
self.assert_dummy_conf()
@inlineCallbacks
def test_destroy_not_installed(self):
"""Check .destroy() does nothing if not installed"""
yield self.service.destroy()
self.assert_no_conf()
@inlineCallbacks
def test_destroy_not_started(self):
"""Check .destroy just deletes conf if not running"""
self.write_dummy_conf()
self.setup_mock()
self.mock_status(succeed("blah stop/waiting blah"))
self.mocker.replay()
yield self.service.destroy()
self.assert_no_conf()
@inlineCallbacks
def test_destroy_started(self):
"""Check .destroy() stops running service and deletes conf file"""
self.write_dummy_conf()
self.setup_mock()
self.mock_status(succeed("blah start/running blah 54321"))
self.mock_stop()
self.mocker.replay()
yield self.service.destroy()
self.assert_no_conf()
@inlineCallbacks
def test_use_sudo(self):
"""Check that expected commands are generated when use_sudo is set"""
self.setup_mock()
self.service = UpstartService("some-name", use_sudo=True)
self.setup_service()
with open(self.output, "w") as f:
f.write("clear this file out...")
def verify_cp(args, **kwargs):
sudo, cp, src, dst = args
self.assertEquals(sudo, "sudo")
self.assertEquals(cp, "cp")
with open(os.path.join(DATA_DIR, "test_standard_install")) as exp:
with open(src) as actual:
self.assertEquals(actual.read(), exp.read())
self.assertEquals(dst, self.conf)
self.write_dummy_conf()
self.check_call(ANY, KWARGS)
self.mocker.call(verify_cp)
self.mock_call(("sudo", "rm", self.output))
self.mock_call(("sudo", "chmod", "644", self.conf))
self.mock_status(succeed("blah stop/waiting blah"))
self.mock_call(("sudo", "/sbin/start", "some-name"))
# 5 for initial stability check; 1 for final do-we-need-to-stop check
for _ in range(6):
self.mock_status(succeed("blah start/running blah 12345"))
self.mock_call(("sudo", "/sbin/stop", "some-name"))
self.mock_call(("sudo", "rm", self.conf))
self.mock_call(("sudo", "rm", self.output))
self.mocker.replay()
yield self.service.start()
yield self.service.destroy()
class StorageServer(object):
def __init__(self, juju_unit_namespace, storage_dir=None,
host=None, port=None, logfile=None):
"""Management facade for a web server on top of the provider storage.
:param juju_unit_namespace: For disambiguation.
:param host: Host interface to bind to.
:param port: Port to bind to.
:param logfile: Path to store log output.
"""
if storage_dir:
storage_dir = os.path.abspath(storage_dir)
self._storage_dir = storage_dir
self._host = host
self._port = port
self._logfile = logfile
self._service = UpstartService(
"juju-%s-file-storage" % juju_unit_namespace, use_sudo=True)
self._service.set_description(
"Juju file storage for %s" % juju_unit_namespace)
self._service_args = [
"twistd",
"--nodaemon",
"--uid", str(os.getuid()),
"--gid", str(os.getgid()),
"--logfile", logfile,
"--pidfile=",
"-d", self._storage_dir,
"web",
"--port", "tcp:%s:interface=%s" % (self._port, self._host),
"--path", self._storage_dir]
@inlineCallbacks
def is_serving(self):
try:
storage = LocalStorage(self._storage_dir)
yield getPage((yield storage.get_url(SERVER_URL_KEY)))
returnValue(True)
except ConnectionRefusedError:
returnValue(False)
@inlineCallbacks
def start(self):
"""Start the storage server.
Also stores the storage server url directly into provider storage.
"""
assert self._storage_dir, "no storage_dir set"
assert self._host, "no host set"
assert self._port, "no port set"
assert None not in self._service_args, "unset params"
assert os.path.exists(self._storage_dir), "Invalid storage directory"
try:
with open(self._logfile, "a"):
pass
except IOError:
raise AssertionError("logfile not writable by this user")
storage = LocalStorage(self._storage_dir)
yield storage.put(
SERVER_URL_KEY,
StringIO(yaml.safe_dump(
{"storage-url": "http://%s:%s/" % (self._host, self._port)})))
self._service.set_command(" ".join(self._service_args))
yield self._service.start()
def get_pid(self):
return self._service.get_pid()
def stop(self):
"""Stop the storage server."""
return self._service.destroy()
class UnitMachineDeployment(object):
""" Deploy a unit directly onto a machine.
A service unit deployed directly to a machine has full access
to the machine resources.
Units deployed in such a manner have no isolation from other units
on the machine, and may leave artifacts on the machine even upon
service destruction.
"""
unit_agent_module = "juju.agents.unit"
def __init__(self, unit_name, juju_home):
assert ".." not in unit_name, "Invalid Unit Name"
self.unit_name = unit_name
self.juju_home = juju_home
self.unit_path_name = unit_name.replace("/", "-")
self.directory = os.path.join(
self.juju_home, "units", self.unit_path_name)
self.service = UpstartService(
# NOTE: we need use_sudo to work correctly during tests that
# launch actual processes (rather than just mocking/trusting).
"juju-%s" % self.unit_path_name, use_sudo=True)
@inlineCallbacks
def start(self, machine_id, zookeeper_hosts, bundle):
"""Start a service unit agent."""
self.unpack_charm(bundle)
self.service.set_description(
"Juju unit agent for %s" % self.unit_name)
self.service.set_environ(_get_environment(
self.unit_name, self.juju_home, machine_id, zookeeper_hosts))
self.service.set_command(" ".join((
"/usr/bin/python", "-m", self.unit_agent_module,
"--nodaemon",
"--logfile", os.path.join(self.directory, "charm.log"),
"--session-file",
"/var/run/juju/unit-%s-agent.zksession" % self.unit_path_name)))
try:
yield self.service.start()
except ServiceError as e:
raise UnitDeploymentError(str(e))
@inlineCallbacks
def destroy(self):
"""Forcibly terminate a service unit agent, and clean disk state.
This will destroy/unmount any state on disk.
"""
yield self.service.destroy()
if os.path.exists(self.directory):
shutil.rmtree(self.directory)
def get_pid(self):
"""Get the service unit's process id."""
return self.service.get_pid()
def is_running(self):
"""Is the service unit running."""
return self.service.is_running()
def unpack_charm(self, charm):
"""Unpack a charm to the service units directory."""
if not isinstance(charm, CharmBundle):
raise UnitDeploymentError(
"Invalid charm for deployment: %s" % charm.path)
charm.extract_to(os.path.join(self.directory, "charm"))
def start(self, machine_id, zookeeper_hosts, bundle):
"""Start the unit.
Creates and starts an lxc container for the unit.
"""
# remove any quoting around the key
public_key = os.environ.get("JUJU_PUBLIC_KEY", "")
public_key = public_key.strip("'\"")
# Build a template container that can be cloned in deploy
# we leave the loosely initialized self.container in place for
# the class as thats all we need for methods other than start.
self.container, self.directory = yield self._get_container(
machine_id, bundle, public_key)
# Create state directories for unit in the container
self.setup_directories()
# Extract the charm bundle
charm_path = os.path.join(
self.directory, "var", "lib", "juju", "units",
self.unit_path_name, "charm")
bundle.extract_to(charm_path)
log.debug("Charm extracted into container")
# Write upstart file for the agent into the container
service_name = "juju-%s" % self.unit_path_name
init_dir = os.path.join(self.directory, "etc", "init")
service = UpstartService(service_name, init_dir=init_dir)
service.set_description(
"Juju unit agent for %s" % self.unit_name)
service.set_environ(_get_environment(
self.unit_name, "/var/lib/juju", machine_id, zookeeper_hosts))
service.set_output_path(
"/var/log/juju/unit-%s-output.log" % self.unit_path_name)
service.set_command(" ".join((
"/usr/bin/python",
"-m", "juju.agents.unit",
"--nodaemon",
"--logfile", "/var/log/juju/unit-%s.log" % self.unit_path_name,
"--session-file",
"/var/run/juju/unit-%s-agent.zksession" % self.unit_path_name)))
yield service.install()
# Create symlinks on the host for easier access to the unit log files
unit_log_path_host = os.path.join(
self.juju_home, "units", self.unit_path_name, "unit.log")
if not os.path.lexists(unit_log_path_host):
os.symlink(
os.path.join(self.directory, "var", "log", "juju",
"unit-%s.log" % self.unit_path_name),
unit_log_path_host)
unit_output_path_host = os.path.join(
self.juju_home, "units", self.unit_path_name, "output.log")
if not os.path.lexists(unit_output_path_host):
os.symlink(
os.path.join(self.directory, "var", "log", "juju",
"unit-%s-output.log" % self.unit_path_name),
unit_output_path_host)
# Debug log for the container
container_log_path = os.path.join(
self.juju_home, "units", self.unit_path_name, "container.log")
self.container.debug_log = container_log_path
log.debug("Starting container...")
yield self.container.run()
log.info("Started container for %s", self.unit_name)