def test_set_revision(self):
dir_ = self.copy_charm()
charm = CharmDirectory(dir_)
charm.set_revision(123)
self.assertEquals(charm.get_revision(), 123)
with open(os.path.join(dir_, "revision")) as f:
self.assertEquals(f.read(), "123\n")
def test_no_revision(self):
dir_ = self.copy_charm()
self.delete_revision(dir_)
charm = CharmDirectory(dir_)
self.assertEquals(charm.get_revision(), 0)
with open(os.path.join(dir_, "revision")) as f:
self.assertEquals(f.read(), "0\n")
def setUp(self):
yield super(MachineAgentTest, self).setUp()
self.output = self.capture_logging("juju.agents.machine",
level=logging.DEBUG)
config = self.get_test_environment_config()
environment = config.get_default()
# Store the environment to zookeeper
environment_state_manager = EnvironmentStateManager(self.client)
yield environment_state_manager.set_config_state(config, "myfirstenv")
# Load the environment with the charm state and charm binary
self.provider = environment.get_machine_provider()
self.storage = self.provider.get_file_storage()
self.charm = CharmDirectory(self.sample_dir1)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.publisher.add_charm(local_charm_id(self.charm), self.charm)
charm_states = yield self.publisher.publish()
self.charm_state = charm_states[0]
# Create a service from the charm from which we can create units for
# the machine.
self.service_state_manager = ServiceStateManager(self.client)
self.service = yield self.service_state_manager.add_service_state(
"fatality-blog", self.charm_state)
def test_extract_symlink_mode(self):
# lp:973260 - charms packed by different tools that record symlink
# mode permissions differently (ie the charm store) don't extract
# correctly.
charm_path = self.copy_charm()
sym_path = os.path.join(charm_path, 'foobar')
os.symlink('metadata.yaml', sym_path)
charm_dir = CharmDirectory(charm_path)
normal_path = charm_dir.as_bundle().path
zf_src = zipfile.ZipFile(normal_path, "r")
foreign_path = os.path.join(self.makeDir(), "store.charm")
zf_dst = zipfile.ZipFile(foreign_path, "w")
for info in zf_src.infolist():
if info.filename == "foobar":
# This is what the charm store does:
info.external_attr = (stat.S_IFLNK | 0777) << 16
zf_dst.writestr(info, zf_src.read(info.filename))
zf_src.close()
zf_dst.close()
bundle = CharmBundle(foreign_path)
extract_dir = self.makeDir()
bundle.extract_to(extract_dir)
self.assertIn("foobar", os.listdir(extract_dir))
self.assertTrue(os.path.islink(os.path.join(extract_dir, "foobar")))
self.assertEqual(os.readlink(os.path.join(extract_dir, 'foobar')),
'metadata.yaml')
def test_no_revision(self):
dir_ = self.copy_charm()
self.delete_revision(dir_)
charm = CharmDirectory(dir_)
self.assertEquals(charm.get_revision(), 0)
with open(os.path.join(dir_, "revision")) as f:
self.assertEquals(f.read(), "0\n")
def assert_charm_upgraded(self, expect_upgraded):
charm_id = yield self.states["unit"].get_charm_id()
self.assertEquals(charm_id == self.expected_upgrade, expect_upgraded)
if expect_upgraded:
expect_revision = CharmURL.parse(self.expected_upgrade).revision
charm = CharmDirectory(os.path.join(self.unit_directory, "charm"))
self.assertEquals(charm.get_revision(), expect_revision)
def test_set_revision(self):
dir_ = self.copy_charm()
charm = CharmDirectory(dir_)
charm.set_revision(123)
self.assertEquals(charm.get_revision(), 123)
with open(os.path.join(dir_, "revision")) as f:
self.assertEquals(f.read(), "123\n")
def test_competing_revisions(self):
dir_ = self.copy_charm()
self.set_metadata_revision(dir_, 999)
log = self.capture_logging("juju.charm")
charm = CharmDirectory(dir_)
self.assertEquals(charm.get_revision(), 1)
self.assertIn(
"revision field is obsolete. Move it to the 'revision' file.",
log.getvalue())
def test_competing_revisions(self):
dir_ = self.copy_charm()
self.set_metadata_revision(dir_, 999)
log = self.capture_logging("juju.charm")
charm = CharmDirectory(dir_)
self.assertEquals(charm.get_revision(), 1)
self.assertIn(
"revision field is obsolete. Move it to the 'revision' file.",
log.getvalue())
def test_charm_base_inheritance(self):
"""
get_sha256() should be implemented in the base class,
and should use compute_sha256 to calculate the digest.
"""
directory = CharmDirectory(self.sample_dir1)
bundle = directory.as_bundle()
digest = compute_file_hash(hashlib.sha256, bundle.path)
self.assertEquals(digest, directory.get_sha256())
def test_charm_base_inheritance(self):
"""
get_sha256() should be implemented in the base class,
and should use compute_sha256 to calculate the digest.
"""
directory = CharmDirectory(self.sample_dir1)
bundle = directory.as_bundle()
digest = compute_file_hash(hashlib.sha256, bundle.path)
self.assertEquals(digest, directory.get_sha256())
def test_executable_extraction(self):
sample_directory = os.path.join(
repository_directory, "series", "varnish-alternative")
charm_directory = CharmDirectory(sample_directory)
source_hook_path = os.path.join(sample_directory, "hooks", "install")
self.assertTrue(os.access(source_hook_path, os.X_OK))
bundle = charm_directory.as_bundle()
directory = bundle.as_directory()
hook_path = os.path.join(directory.path, "hooks", "install")
self.assertTrue(os.access(hook_path, os.X_OK))
def test_executable_extraction(self):
sample_directory = os.path.join(repository_directory, "series",
"varnish-alternative")
charm_directory = CharmDirectory(sample_directory)
source_hook_path = os.path.join(sample_directory, "hooks", "install")
self.assertTrue(os.access(source_hook_path, os.X_OK))
bundle = charm_directory.as_bundle()
directory = bundle.as_directory()
hook_path = os.path.join(directory.path, "hooks", "install")
self.assertTrue(os.access(hook_path, os.X_OK))
def test_add_charm_with_concurrent(self):
"""
Publishing a charm, that has become published concurrent, after the
add_charm, works fine. it will write to storage regardless. The use
of a sha256 as part of the storage key is utilized to help ensure
uniqueness of bits. The sha256 is also stored with the charm state.
This relation betewen the charm state and the binary bits, helps
guarantee the property that any published charm in zookeeper will use
the binary bits that it was published with.
"""
yield self.publisher.add_charm(self.charm_id, self.charm)
concurrent_publisher = CharmPublisher(self.client, self.storage)
charm = CharmDirectory(self.sample_dir1)
yield concurrent_publisher.add_charm(self.charm_id, charm)
yield self.publisher.publish()
# modify the charm to create a conflict scenario
self.makeFile("zebra", path=os.path.join(self.sample_dir1, "junk.txt"))
# assert the charm now has a different sha post modification
modified_charm_sha = charm.get_sha256()
self.assertNotEqual(modified_charm_sha, self.charm.get_sha256())
# verify publishing raises a stateerror
def verify_failure(result):
if not isinstance(result, Failure):
self.fail("Should have raised state error")
result.trap(StateChanged)
return True
yield concurrent_publisher.publish().addBoth(verify_failure)
# verify the zk state
charm_nodes = yield self.client.get_children("/charms")
self.assertEqual(charm_nodes, [self.charm_key])
content, stat = yield self.client.get("/charms/%s" % charm_nodes[0])
# assert the checksum matches the initially published checksum
self.assertEqual(yaml.load(content)["sha256"], self.charm.get_sha256())
store_path = os.path.join(self.storage_dir, self.charm_storage_key)
self.assertTrue(os.path.exists(store_path))
# and the binary bits where stored
modified_charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, modified_charm_sha))
modified_store_path = os.path.join(self.storage_dir,
modified_charm_storage_key)
self.assertTrue(os.path.exists(modified_store_path))
def test_compute_sha256(self):
"""
Computing the sha256 of a directory will use the bundled
charm, since the hash of the file itself is needed.
"""
directory = CharmDirectory(self.sample_dir1)
sha256 = directory.compute_sha256()
charm_bundle = directory.as_bundle()
self.assertEquals(type(charm_bundle), CharmBundle)
self.assertEquals(compute_file_hash(hashlib.sha256, charm_bundle.path),
sha256)
def test_compute_sha256(self):
"""
Computing the sha256 of a directory will use the bundled
charm, since the hash of the file itself is needed.
"""
directory = CharmDirectory(self.sample_dir1)
sha256 = directory.compute_sha256()
charm_bundle = directory.as_bundle()
self.assertEquals(type(charm_bundle), CharmBundle)
self.assertEquals(compute_file_hash(hashlib.sha256,
charm_bundle.path),
sha256)
def test_as_bundle_file_lifetime(self):
"""
The temporary bundle file created should have a life time
equivalent to that of the directory object itself.
"""
directory = CharmDirectory(self.sample_dir1)
charm_bundle = directory.as_bundle()
gc.collect()
self.assertTrue(os.path.isfile(charm_bundle.path))
del directory
gc.collect()
self.assertFalse(os.path.isfile(charm_bundle.path))
def test_as_bundle_file_lifetime(self):
"""
The temporary bundle file created should have a life time
equivalent to that of the directory object itself.
"""
directory = CharmDirectory(self.sample_dir1)
charm_bundle = directory.as_bundle()
gc.collect()
self.assertTrue(os.path.isfile(charm_bundle.path))
del directory
gc.collect()
self.assertFalse(os.path.isfile(charm_bundle.path))
def setUp(self):
super(LocalRepositoryTest, self).setUp()
# bundle sample charms
CharmDirectory(self.sample_dir1).make_archive(
os.path.join(self.bundled_repo_path, "series", "old.charm"))
CharmDirectory(self.sample_dir2).make_archive(
os.path.join(self.bundled_repo_path, "series", "new.charm"))
# define repository objects
self.repository1 = LocalCharmRepository(self.unbundled_repo_path)
self.repository2 = LocalCharmRepository(self.bundled_repo_path)
def test_as_bundle_with_relative_path(self):
"""
Ensure that as_bundle works correctly with relative paths.
"""
current_dir = os.getcwd()
os.chdir(self.sample_dir2)
self.addCleanup(os.chdir, current_dir)
charm_dir = "../%s" % os.path.basename(self.sample_dir1)
directory = CharmDirectory(charm_dir)
charm_bundle = directory.as_bundle()
self.assertEquals(type(charm_bundle), CharmBundle)
self.assertEquals(charm_bundle.metadata.name, "sample")
def test_as_bundle_with_relative_path(self):
"""
Ensure that as_bundle works correctly with relative paths.
"""
current_dir = os.getcwd()
os.chdir(self.sample_dir2)
self.addCleanup(os.chdir, current_dir)
charm_dir = "../%s" % os.path.basename(self.sample_dir1)
directory = CharmDirectory(charm_dir)
charm_bundle = directory.as_bundle()
self.assertEquals(type(charm_bundle), CharmBundle)
self.assertEquals(charm_bundle.metadata.name, "sample")
def test_as_bundle(self):
directory = CharmDirectory(self.sample_dir1)
charm_bundle = directory.as_bundle()
self.assertEquals(type(charm_bundle), CharmBundle)
self.assertEquals(charm_bundle.metadata.name, "sample")
self.assertIn("sample-1.charm", charm_bundle.path)
total_compressed = 0
total_uncompressed = 0
zip_file = zipfile.ZipFile(charm_bundle.path)
for n in zip_file.namelist():
info = zip_file.getinfo(n)
total_compressed += info.compress_size
total_uncompressed += info.file_size
self.assertTrue(total_compressed < total_uncompressed)
def test_as_bundle(self):
directory = CharmDirectory(self.sample_dir1)
charm_bundle = directory.as_bundle()
self.assertEquals(type(charm_bundle), CharmBundle)
self.assertEquals(charm_bundle.metadata.name, "sample")
self.assertIn("sample-1.charm", charm_bundle.path)
total_compressed = 0
total_uncompressed = 0
zip_file = zipfile.ZipFile(charm_bundle.path)
for n in zip_file.namelist():
info = zip_file.getinfo(n)
total_compressed += info.compress_size
total_uncompressed += info.file_size
self.assertTrue(total_compressed < total_uncompressed)
def setUp(self):
yield super(RemoteUpgradeCharmTest, self).setUp()
config = {
"environments": {"firstenv": {"type": "dummy"}}}
self.write_config(dump(config))
self.config.load()
charm = CharmDirectory(os.path.join(
test_repository_path, "series", "mysql"))
self.charm_state_manager.add_charm_state(
"cs:series/mysql-1", charm, "")
self.service_state1 = yield self.add_service_from_charm(
"mysql", "cs:series/mysql-1")
self.service_unit1 = yield self.service_state1.add_unit_state()
self.unit1_workflow = UnitWorkflowState(
self.client, self.service_unit1, None, self.makeDir())
yield self.unit1_workflow.set_state("started")
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
self.output = self.capture_logging()
self.stderr = self.capture_stream("stderr")
def setUp(self):
yield super(MachineAgentTest, self).setUp()
self.output = self.capture_logging(
"juju.agents.machine", level=logging.DEBUG)
config = self.get_test_environment_config()
environment = config.get_default()
# Store the environment to zookeeper
environment_state_manager = EnvironmentStateManager(self.client)
yield environment_state_manager.set_config_state(config, "myfirstenv")
# Load the environment with the charm state and charm binary
self.provider = environment.get_machine_provider()
self.storage = self.provider.get_file_storage()
self.charm = CharmDirectory(self.sample_dir1)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.publisher.add_charm(local_charm_id(self.charm), self.charm)
charm_states = yield self.publisher.publish()
self.charm_state = charm_states[0]
# Create a service from the charm from which we can create units for
# the machine.
self.service_state_manager = ServiceStateManager(self.client)
self.service = yield self.service_state_manager.add_service_state(
"fatality-blog", self.charm_state)
def test_make_archive(self):
# make archive from sample directory
directory = CharmDirectory(sample_directory)
f = tempfile.NamedTemporaryFile(suffix=".charm")
directory.make_archive(f.name)
# open archive in .zip-format and assert integrity
from zipfile import ZipFile
zf = ZipFile(f.name)
self.assertEqual(zf.testzip(), None)
# assert included
included = [info.filename for info in zf.infolist()]
self.assertEqual(
set(included),
set(("metadata.yaml", "empty/", "src/", "src/hello.c",
"config.yaml", "hooks/", "hooks/install", "revision")))
def test_make_archive(self):
# make archive from sample directory
directory = CharmDirectory(sample_directory)
f = tempfile.NamedTemporaryFile(suffix=".charm")
directory.make_archive(f.name)
# open archive in .zip-format and assert integrity
from zipfile import ZipFile
zf = ZipFile(f.name)
self.assertEqual(zf.testzip(), None)
# assert included
included = [info.filename for info in zf.infolist()]
self.assertEqual(
set(included),
set(("metadata.yaml", "empty/", "src/", "src/hello.c",
"config.yaml", "hooks/", "hooks/install", "revision")))
def setUp(self):
super(CharmPublisherTest, self).setUp()
zookeeper.set_debug_level(0)
self.charm = CharmDirectory(self.sample_dir1)
self.charm_id = local_charm_id(self.charm)
self.charm_key = under.quote(self.charm_id)
# provider storage key
self.charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, self.charm.get_sha256()))
self.client = ZookeeperClient(get_test_zookeeper_address())
self.storage_dir = self.makeDir()
self.storage = FileStorage(self.storage_dir)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.client.connect()
yield self.client.create("/charms")
def setUp(self):
super(RemoteRepositoryTest, self).setUp()
self.cache_path = os.path.join(tempfile.mkdtemp(), "notexistyet")
self.download_path = os.path.join(self.cache_path, "downloads")
def delete():
if os.path.exists(self.cache_path):
shutil.rmtree(self.cache_path)
self.addCleanup(delete)
self.charm = CharmDirectory(
os.path.join(self.unbundled_repo_path, "series", "dummy"))
with open(self.charm.as_bundle().path, "rb") as f:
self.bundle_data = f.read()
self.sha256 = self.charm.as_bundle().get_sha256()
self.getPage = self.mocker.replace("twisted.web.client.getPage")
self.downloadPage = self.mocker.replace(
"twisted.web.client.downloadPage")
def setUp(self):
yield super(StateTestBase, self).setUp()
zookeeper.set_debug_level(0)
self.charm = CharmDirectory(sample_directory)
self.client = self.get_zookeeper_client()
yield self.client.connect()
yield self.client.create("/charms")
yield self.client.create("/machines")
yield self.client.create("/services")
yield self.client.create("/units")
yield self.client.create("/relations")
def test_extract_symlink(self):
extract_dir = self.makeDir()
charm_path = self.copy_charm()
sym_path = os.path.join(charm_path, 'foobar')
os.symlink('metadata.yaml', sym_path)
charm_dir = CharmDirectory(charm_path)
bundle = charm_dir.as_bundle()
bundle.extract_to(extract_dir)
self.assertIn("foobar", os.listdir(extract_dir))
self.assertTrue(os.path.islink(os.path.join(extract_dir, "foobar")))
self.assertEqual(os.readlink(os.path.join(extract_dir, 'foobar')),
'metadata.yaml')
# Verify we can extract it over again
os.remove(sym_path)
os.symlink('./config.yaml', sym_path)
charm_dir = CharmDirectory(charm_path)
bundle = charm_dir.as_bundle()
bundle.extract_to(extract_dir)
self.assertEqual(os.readlink(os.path.join(extract_dir, 'foobar')),
'./config.yaml')
def add_charm(self, metadata, revision, repository_dir=None, bundle=False):
if repository_dir is None:
repository_dir = self.makeDir()
series_dir = os.path.join(repository_dir, "series")
os.mkdir(series_dir)
charm_dir = self.makeDir()
with open(os.path.join(charm_dir, "metadata.yaml"), "w") as f:
f.write(dump(metadata))
with open(os.path.join(charm_dir, "revision"), "w") as f:
f.write(str(revision))
if bundle:
CharmDirectory(charm_dir).make_archive(
os.path.join(series_dir, "%s.charm" % metadata["name"]))
else:
os.rename(charm_dir, os.path.join(series_dir, metadata["name"]))
return LocalCharmRepository(repository_dir)
def setUp(self):
super(CharmPublisherTest, self).setUp()
zookeeper.set_debug_level(0)
self.charm = CharmDirectory(self.sample_dir1)
self.charm_id = local_charm_id(self.charm)
self.charm_key = under.quote(self.charm_id)
# provider storage key
self.charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, self.charm.get_sha256()))
self.client = ZookeeperClient(get_test_zookeeper_address())
self.storage_dir = self.makeDir()
self.storage = FileStorage(self.storage_dir)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.client.connect()
yield self.client.create("/charms")
def setUp(self):
super(RemoteRepositoryTest, self).setUp()
self.cache_path = os.path.join(
self.makeDir(), "notexistyet")
self.download_path = os.path.join(self.cache_path, "downloads")
def delete():
if os.path.exists(self.cache_path):
shutil.rmtree(self.cache_path)
self.addCleanup(delete)
self.charm = CharmDirectory(
os.path.join(self.unbundled_repo_path, "series", "dummy"))
with open(self.charm.as_bundle().path, "rb") as f:
self.bundle_data = f.read()
self.sha256 = self.charm.as_bundle().get_sha256()
self.getPage = self.mocker.replace("twisted.web.client.getPage")
self.downloadPage = self.mocker.replace(
"twisted.web.client.downloadPage")
def test_add_charm_with_concurrent(self):
"""
Publishing a charm, that has become published concurrent, after the
add_charm, works fine. it will write to storage regardless. The use
of a sha256 as part of the storage key is utilized to help ensure
uniqueness of bits. The sha256 is also stored with the charm state.
This relation betewen the charm state and the binary bits, helps
guarantee the property that any published charm in zookeeper will use
the binary bits that it was published with.
"""
yield self.publisher.add_charm(self.charm_id, self.charm)
concurrent_publisher = CharmPublisher(
self.client, self.storage)
charm = CharmDirectory(self.sample_dir1)
yield concurrent_publisher.add_charm(self.charm_id, charm)
yield self.publisher.publish()
# modify the charm to create a conflict scenario
self.makeFile("zebra",
path=os.path.join(self.sample_dir1, "junk.txt"))
# assert the charm now has a different sha post modification
modified_charm_sha = charm.get_sha256()
self.assertNotEqual(
modified_charm_sha,
self.charm.get_sha256())
# verify publishing raises a stateerror
def verify_failure(result):
if not isinstance(result, Failure):
self.fail("Should have raised state error")
result.trap(StateChanged)
return True
yield concurrent_publisher.publish().addBoth(verify_failure)
# verify the zk state
charm_nodes = yield self.client.get_children("/charms")
self.assertEqual(charm_nodes, [self.charm_key])
content, stat = yield self.client.get(
"/charms/%s" % charm_nodes[0])
# assert the checksum matches the initially published checksum
self.assertEqual(
yaml.load(content)["sha256"], self.charm.get_sha256())
store_path = os.path.join(self.storage_dir, self.charm_storage_key)
self.assertTrue(os.path.exists(store_path))
# and the binary bits where stored
modified_charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, modified_charm_sha))
modified_store_path = os.path.join(
self.storage_dir, modified_charm_storage_key)
self.assertTrue(os.path.exists(modified_store_path))
class CharmPublisherTest(RepositoryTestBase):
@inlineCallbacks
def setUp(self):
super(CharmPublisherTest, self).setUp()
zookeeper.set_debug_level(0)
self.charm = CharmDirectory(self.sample_dir1)
self.charm_id = local_charm_id(self.charm)
self.charm_key = under.quote(self.charm_id)
# provider storage key
self.charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, self.charm.get_sha256()))
self.client = ZookeeperClient(get_test_zookeeper_address())
self.storage_dir = self.makeDir()
self.storage = FileStorage(self.storage_dir)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.client.connect()
yield self.client.create("/charms")
def tearDown(self):
deleteTree("/", self.client.handle)
self.client.close()
super(CharmPublisherTest, self).tearDown()
@inlineCallbacks
def test_add_charm_and_publish(self):
open_file_count = _count_open_files()
yield self.publisher.add_charm(self.charm_id, self.charm)
result = yield self.publisher.publish()
self.assertEquals(_count_open_files(), open_file_count)
children = yield self.client.get_children("/charms")
self.assertEqual(children, [self.charm_key])
fh = yield self.storage.get(self.charm_storage_key)
bundle = CharmBundle(fh)
self.assertEqual(self.charm.get_sha256(), bundle.get_sha256())
self.assertEqual(
result[0].bundle_url, "file://%s/%s" % (
self.storage_dir, self.charm_storage_key))
@inlineCallbacks
def test_published_charm_sans_unicode(self):
yield self.publisher.add_charm(self.charm_id, self.charm)
yield self.publisher.publish()
data, stat = yield self.client.get("/charms/%s" % self.charm_key)
self.assertNotIn("unicode", data)
@inlineCallbacks
def test_add_charm_with_concurrent(self):
"""
Publishing a charm, that has become published concurrent, after the
add_charm, works fine. it will write to storage regardless. The use
of a sha256 as part of the storage key is utilized to help ensure
uniqueness of bits. The sha256 is also stored with the charm state.
This relation betewen the charm state and the binary bits, helps
guarantee the property that any published charm in zookeeper will use
the binary bits that it was published with.
"""
yield self.publisher.add_charm(self.charm_id, self.charm)
concurrent_publisher = CharmPublisher(
self.client, self.storage)
charm = CharmDirectory(self.sample_dir1)
yield concurrent_publisher.add_charm(self.charm_id, charm)
yield self.publisher.publish()
# modify the charm to create a conflict scenario
self.makeFile("zebra",
path=os.path.join(self.sample_dir1, "junk.txt"))
# assert the charm now has a different sha post modification
modified_charm_sha = charm.get_sha256()
self.assertNotEqual(
modified_charm_sha,
self.charm.get_sha256())
# verify publishing raises a stateerror
def verify_failure(result):
if not isinstance(result, Failure):
self.fail("Should have raised state error")
result.trap(StateChanged)
return True
yield concurrent_publisher.publish().addBoth(verify_failure)
# verify the zk state
charm_nodes = yield self.client.get_children("/charms")
self.assertEqual(charm_nodes, [self.charm_key])
content, stat = yield self.client.get(
"/charms/%s" % charm_nodes[0])
# assert the checksum matches the initially published checksum
self.assertEqual(
yaml.load(content)["sha256"], self.charm.get_sha256())
store_path = os.path.join(self.storage_dir, self.charm_storage_key)
self.assertTrue(os.path.exists(store_path))
# and the binary bits where stored
modified_charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, modified_charm_sha))
modified_store_path = os.path.join(
self.storage_dir, modified_charm_storage_key)
self.assertTrue(os.path.exists(modified_store_path))
@inlineCallbacks
def test_add_charm_with_concurrent_removal(self):
"""
If a charm is published, and it detects that the charm exists
already exists, it will attempt to retrieve the charm state to
verify there is no checksum mismatch. If concurrently the charm
is removed, the publisher should fail with a statechange error.
"""
manager = self.mocker.patch(CharmStateManager)
manager.get_charm_state(self.charm_id)
self.mocker.passthrough()
def match_charm_bundle(bundle):
return isinstance(bundle, CharmBundle)
def match_charm_url(url):
return url.startswith("file://")
manager.add_charm_state(
self.charm_id, MATCH(match_charm_bundle), MATCH(match_charm_url))
self.mocker.result(fail(zookeeper.NodeExistsException()))
manager.get_charm_state(self.charm_id)
self.mocker.result(fail(zookeeper.NoNodeException()))
self.mocker.replay()
yield self.publisher.add_charm(self.charm_id, self.charm)
yield self.failUnlessFailure(self.publisher.publish(), StateChanged)
@inlineCallbacks
def test_add_charm_already_known(self):
"""Adding an existing charm, is an effective noop, as its not added
to the internal publisher queue.
"""
# Do an initial publishing of the charm
scheduled = yield self.publisher.add_charm(self.charm_id, self.charm)
self.assertTrue(scheduled)
result = yield self.publisher.publish()
self.assertEqual(result[0].name, self.charm.metadata.name)
publisher = CharmPublisher(self.client, self.storage)
scheduled = yield publisher.add_charm(self.charm_id, self.charm)
self.assertFalse(scheduled)
scheduled = yield publisher.add_charm(self.charm_id, self.charm)
self.assertFalse(scheduled)
result = yield publisher.publish()
self.assertEqual(result[0].name, self.charm.metadata.name)
self.assertEqual(result[1].name, self.charm.metadata.name)
def setUp(self):
directory = CharmDirectory(sample_directory)
# add sample directory
self.filename = self.makeFile(suffix=".charm")
directory.make_archive(self.filename)
class CharmPublisherTest(RepositoryTestBase):
@inlineCallbacks
def setUp(self):
super(CharmPublisherTest, self).setUp()
zookeeper.set_debug_level(0)
self.charm = CharmDirectory(self.sample_dir1)
self.charm_id = local_charm_id(self.charm)
self.charm_key = under.quote(self.charm_id)
# provider storage key
self.charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, self.charm.get_sha256()))
self.client = ZookeeperClient(get_test_zookeeper_address())
self.storage_dir = self.makeDir()
self.storage = FileStorage(self.storage_dir)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.client.connect()
yield self.client.create("/charms")
def tearDown(self):
deleteTree("/", self.client.handle)
self.client.close()
super(CharmPublisherTest, self).tearDown()
@inlineCallbacks
def test_add_charm_and_publish(self):
open_file_count = _count_open_files()
yield self.publisher.add_charm(self.charm_id, self.charm)
result = yield self.publisher.publish()
self.assertEquals(_count_open_files(), open_file_count)
children = yield self.client.get_children("/charms")
self.assertEqual(children, [self.charm_key])
fh = yield self.storage.get(self.charm_storage_key)
bundle = CharmBundle(fh)
self.assertEqual(self.charm.get_sha256(), bundle.get_sha256())
self.assertEqual(
result[0].bundle_url,
"file://%s/%s" % (self.storage_dir, self.charm_storage_key))
@inlineCallbacks
def test_published_charm_sans_unicode(self):
yield self.publisher.add_charm(self.charm_id, self.charm)
yield self.publisher.publish()
data, stat = yield self.client.get("/charms/%s" % self.charm_key)
self.assertNotIn("unicode", data)
@inlineCallbacks
def test_add_charm_with_concurrent(self):
"""
Publishing a charm, that has become published concurrent, after the
add_charm, works fine. it will write to storage regardless. The use
of a sha256 as part of the storage key is utilized to help ensure
uniqueness of bits. The sha256 is also stored with the charm state.
This relation betewen the charm state and the binary bits, helps
guarantee the property that any published charm in zookeeper will use
the binary bits that it was published with.
"""
yield self.publisher.add_charm(self.charm_id, self.charm)
concurrent_publisher = CharmPublisher(self.client, self.storage)
charm = CharmDirectory(self.sample_dir1)
yield concurrent_publisher.add_charm(self.charm_id, charm)
yield self.publisher.publish()
# modify the charm to create a conflict scenario
self.makeFile("zebra", path=os.path.join(self.sample_dir1, "junk.txt"))
# assert the charm now has a different sha post modification
modified_charm_sha = charm.get_sha256()
self.assertNotEqual(modified_charm_sha, self.charm.get_sha256())
# verify publishing raises a stateerror
def verify_failure(result):
if not isinstance(result, Failure):
self.fail("Should have raised state error")
result.trap(StateChanged)
return True
yield concurrent_publisher.publish().addBoth(verify_failure)
# verify the zk state
charm_nodes = yield self.client.get_children("/charms")
self.assertEqual(charm_nodes, [self.charm_key])
content, stat = yield self.client.get("/charms/%s" % charm_nodes[0])
# assert the checksum matches the initially published checksum
self.assertEqual(yaml.load(content)["sha256"], self.charm.get_sha256())
store_path = os.path.join(self.storage_dir, self.charm_storage_key)
self.assertTrue(os.path.exists(store_path))
# and the binary bits where stored
modified_charm_storage_key = under.quote(
"%s:%s" % (self.charm_id, modified_charm_sha))
modified_store_path = os.path.join(self.storage_dir,
modified_charm_storage_key)
self.assertTrue(os.path.exists(modified_store_path))
@inlineCallbacks
def test_add_charm_with_concurrent_removal(self):
"""
If a charm is published, and it detects that the charm exists
already exists, it will attempt to retrieve the charm state to
verify there is no checksum mismatch. If concurrently the charm
is removed, the publisher should fail with a statechange error.
"""
manager = self.mocker.patch(CharmStateManager)
manager.get_charm_state(self.charm_id)
self.mocker.passthrough()
def match_charm_bundle(bundle):
return isinstance(bundle, CharmBundle)
def match_charm_url(url):
return url.startswith("file://")
manager.add_charm_state(self.charm_id, MATCH(match_charm_bundle),
MATCH(match_charm_url))
self.mocker.result(fail(zookeeper.NodeExistsException()))
manager.get_charm_state(self.charm_id)
self.mocker.result(fail(zookeeper.NoNodeException()))
self.mocker.replay()
yield self.publisher.add_charm(self.charm_id, self.charm)
yield self.failUnlessFailure(self.publisher.publish(), StateChanged)
@inlineCallbacks
def test_add_charm_already_known(self):
"""Adding an existing charm, is an effective noop, as its not added
to the internal publisher queue.
"""
# Do an initial publishing of the charm
scheduled = yield self.publisher.add_charm(self.charm_id, self.charm)
self.assertTrue(scheduled)
result = yield self.publisher.publish()
self.assertEqual(result[0].name, self.charm.metadata.name)
publisher = CharmPublisher(self.client, self.storage)
scheduled = yield publisher.add_charm(self.charm_id, self.charm)
self.assertFalse(scheduled)
scheduled = yield publisher.add_charm(self.charm_id, self.charm)
self.assertFalse(scheduled)
result = yield publisher.publish()
self.assertEqual(result[0].name, self.charm.metadata.name)
self.assertEqual(result[1].name, self.charm.metadata.name)
def setUp(self):
directory = CharmDirectory(sample_directory)
# add sample directory
self.filename = self.makeFile(suffix=".charm")
directory.make_archive(self.filename)
class RemoteRepositoryTest(RepositoryTestBase):
def setUp(self):
super(RemoteRepositoryTest, self).setUp()
self.cache_path = os.path.join(tempfile.mkdtemp(), "notexistyet")
self.download_path = os.path.join(self.cache_path, "downloads")
def delete():
if os.path.exists(self.cache_path):
shutil.rmtree(self.cache_path)
self.addCleanup(delete)
self.charm = CharmDirectory(
os.path.join(self.unbundled_repo_path, "series", "dummy"))
with open(self.charm.as_bundle().path, "rb") as f:
self.bundle_data = f.read()
self.sha256 = self.charm.as_bundle().get_sha256()
self.getPage = self.mocker.replace("twisted.web.client.getPage")
self.downloadPage = self.mocker.replace(
"twisted.web.client.downloadPage")
def repo(self, url_base):
return RemoteCharmRepository(url_base, self.cache_path)
def cache_location(self, url_str, revision):
charm_url = CharmURL.parse(url_str)
cache_key = under.quote("%s.charm" %
(charm_url.with_revision(revision)))
return os.path.join(self.cache_path, cache_key)
def charm_info(self, url_str, revision, warnings=None, errors=None):
info = {"revision": revision, "sha256": self.sha256}
if errors:
info["errors"] = errors
if warnings:
info["warnings"] = warnings
return json.dumps({url_str: info})
def mock_charm_info(self, url, result):
self.getPage(url)
self.mocker.result(result)
def mock_download(self, url, error=None):
self.downloadPage(url, ANY)
if error:
return self.mocker.result(fail(error))
def download(_, path):
self.assertTrue(path.startswith(self.download_path))
with open(path, "wb") as f:
f.write(self.bundle_data)
return succeed(None)
self.mocker.call(download)
@inlineCallbacks
def assert_find_uncached(self, dns_name, url_str, info_url, find_url):
self.mock_charm_info(info_url, succeed(self.charm_info(url_str, 1)))
self.mock_download(find_url)
self.mocker.replay()
repo = self.repo(dns_name)
charm = yield repo.find(CharmURL.parse(url_str))
self.assertEquals(charm.get_sha256(), self.sha256)
self.assertEquals(charm.path, self.cache_location(url_str, 1))
self.assertEquals(os.listdir(self.download_path), [])
@inlineCallbacks
def assert_find_cached(self, dns_name, url_str, info_url):
os.makedirs(self.cache_path)
cache_location = self.cache_location(url_str, 1)
shutil.copy(self.charm.as_bundle().path, cache_location)
self.mock_charm_info(info_url, succeed(self.charm_info(url_str, 1)))
self.mocker.replay()
repo = self.repo(dns_name)
charm = yield repo.find(CharmURL.parse(url_str))
self.assertEquals(charm.get_sha256(), self.sha256)
self.assertEquals(charm.path, cache_location)
def assert_find_error(self, dns_name, url_str, err_type, message):
self.mocker.replay()
repo = self.repo(dns_name)
d = self.assertFailure(repo.find(CharmURL.parse(url_str)), err_type)
def verify(error):
self.assertEquals(str(error), message)
d.addCallback(verify)
return d
@inlineCallbacks
def assert_latest(self, dns_name, url_str, revision):
self.mocker.replay()
repo = self.repo(dns_name)
result = yield repo.latest(CharmURL.parse(url_str))
self.assertEquals(result, revision)
def assert_latest_error(self, dns_name, url_str, err_type, message):
self.mocker.replay()
repo = self.repo(dns_name)
d = self.assertFailure(repo.latest(CharmURL.parse(url_str)), err_type)
def verify(error):
self.assertEquals(str(error), message)
d.addCallback(verify)
return d
def test_find_plain_uncached(self):
return self.assert_find_uncached(
"https://somewhe.re", "cs:series/name",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name",
"https://somewhe.re/charm/series/name-1")
def test_find_revision_uncached(self):
return self.assert_find_uncached(
"https://somewhe.re", "cs:series/name-1",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name-1",
"https://somewhe.re/charm/series/name-1")
def test_find_user_uncached(self):
return self.assert_find_uncached(
"https://somewhereel.se", "cs:~user/srs/name",
"https://somewhereel.se/charm-info?charms=cs%3A%7Euser/srs/name",
"https://somewhereel.se/charm/%7Euser/srs/name-1")
def test_find_plain_cached(self):
return self.assert_find_cached(
"https://somewhe.re", "cs:series/name",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name")
def test_find_revision_cached(self):
return self.assert_find_cached(
"https://somewhe.re", "cs:series/name-1",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name-1")
def test_find_user_cached(self):
return self.assert_find_cached(
"https://somewhereel.se", "cs:~user/srs/name",
"https://somewhereel.se/charm-info?charms=cs%3A%7Euser/srs/name")
def test_find_info_http_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
fail(Error("500")))
return self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmNotFound,
"Charm 'cs:series/name' not found in repository https://anoth.er")
@inlineCallbacks
def test_find_info_store_warning(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name-1",
succeed(
self.charm_info("cs:series/name-1",
1,
warnings=["omg", "halp"])))
self.mock_download("https://anoth.er/charm/series/name-1")
self.mocker.replay()
repo = self.repo("https://anoth.er")
log = self.capture_logging("juju.charm")
charm = yield repo.find(CharmURL.parse("cs:series/name-1"))
self.assertIn("omg", log.getvalue())
self.assertIn("halp", log.getvalue())
self.assertEquals(charm.get_sha256(), self.sha256)
def test_find_info_store_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name-101",
succeed(
self.charm_info("cs:series/name-101",
101,
errors=["oh", "noes"])))
return self.assert_find_error(
"https://anoth.er", "cs:series/name-101", CharmError,
"Error processing 'cs:series/name-101': oh; noes")
def test_find_info_bad_revision(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name-99",
succeed(self.charm_info("cs:series/name-99", 1)))
return self.assert_find_error("https://anoth.er", "cs:series/name-99",
AssertionError, "bad url revision")
def test_find_download_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(json.dumps({"cs:series/name": {
"revision": 123
}})))
self.mock_download("https://anoth.er/charm/series/name-123",
Error("999"))
return self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmNotFound,
"Charm 'cs:series/name-123' not found in repository "
"https://anoth.er")
def test_find_charm_revision_mismatch(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(json.dumps({"cs:series/name": {
"revision": 99
}})))
self.mock_download("https://anoth.er/charm/series/name-99")
return self.assert_find_error("https://anoth.er", "cs:series/name",
AssertionError, "bad charm revision")
@inlineCallbacks
def test_find_downloaded_hash_mismatch(self):
cache_location = self.cache_location("cs:series/name-1", 1)
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(
json.dumps(
{"cs:series/name": {
"revision": 1,
"sha256": "NO YUO"
}})))
self.mock_download("https://anoth.er/charm/series/name-1")
yield self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmError,
"Error processing 'cs:series/name-1 (downloaded)': SHA256 "
"mismatch")
self.assertFalse(os.path.exists(cache_location))
@inlineCallbacks
def test_find_cached_hash_mismatch(self):
os.makedirs(self.cache_path)
cache_location = self.cache_location("cs:series/name-1", 1)
shutil.copy(self.charm.as_bundle().path, cache_location)
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(
json.dumps(
{"cs:series/name": {
"revision": 1,
"sha256": "NO YUO"
}})))
yield self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmError,
"Error processing 'cs:series/name-1 (cached)': SHA256 mismatch")
self.assertFalse(os.path.exists(cache_location))
def test_latest_plain(self):
self.mock_charm_info(
"https://somewhe.re/charm-info?charms=cs%3Afoo/bar",
succeed(self.charm_info("cs:foo/bar", 99)))
return self.assert_latest("https://somewhe.re", "cs:foo/bar-1", 99)
def test_latest_user(self):
self.mock_charm_info(
"https://somewhereel.se/charm-info?charms=cs%3A%7Efee/foo/bar",
succeed(self.charm_info("cs:~fee/foo/bar", 123)))
return self.assert_latest("https://somewhereel.se", "cs:~fee/foo/bar",
123)
def test_latest_revision(self):
self.mock_charm_info(
"https://somewhereel.se/charm-info?charms=cs%3A%7Efee/foo/bar",
succeed(self.charm_info("cs:~fee/foo/bar", 123)))
return self.assert_latest("https://somewhereel.se",
"cs:~fee/foo/bar-99", 123)
def test_latest_http_error(self):
self.mock_charm_info(
"https://andanoth.er/charm-info?charms=cs%3A%7Eblib/blab/blob",
fail(Error("404")))
return self.assert_latest_error(
"https://andanoth.er", "cs:~blib/blab/blob", CharmNotFound,
"Charm 'cs:~blib/blab/blob' not found in repository "
"https://andanoth.er")
@inlineCallbacks
def test_latest_store_warning(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(
self.charm_info("cs:series/name", 1, warnings=["eww",
"yuck"])))
self.mocker.replay()
repo = self.repo("https://anoth.er")
log = self.capture_logging("juju.charm")
revision = yield repo.latest(CharmURL.parse("cs:series/name-1"))
self.assertIn("eww", log.getvalue())
self.assertIn("yuck", log.getvalue())
self.assertEquals(revision, 1)
def test_latest_store_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(
self.charm_info("cs:series/name", 1, errors=["blam", "dink"])))
return self.assert_latest_error(
"https://anoth.er", "cs:series/name-1", CharmError,
"Error processing 'cs:series/name': blam; dink")
def test_info(self):
directory = CharmDirectory(sample_directory)
self.assertTrue(directory.metadata is not None)
self.assertTrue(isinstance(directory.metadata, MetaData))
self.assertEquals(directory.metadata.name, "dummy")
class MachineAgentTest(AgentTestBase, RepositoryTestBase):
agent_class = MachineAgent
@inlineCallbacks
def setUp(self):
yield super(MachineAgentTest, self).setUp()
self.output = self.capture_logging("juju.agents.machine",
level=logging.DEBUG)
config = self.get_test_environment_config()
environment = config.get_default()
# Store the environment to zookeeper
environment_state_manager = EnvironmentStateManager(self.client)
yield environment_state_manager.set_config_state(config, "myfirstenv")
# Load the environment with the charm state and charm binary
self.provider = environment.get_machine_provider()
self.storage = self.provider.get_file_storage()
self.charm = CharmDirectory(self.sample_dir1)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.publisher.add_charm(local_charm_id(self.charm), self.charm)
charm_states = yield self.publisher.publish()
self.charm_state = charm_states[0]
# Create a service from the charm from which we can create units for
# the machine.
self.service_state_manager = ServiceStateManager(self.client)
self.service = yield self.service_state_manager.add_service_state(
"fatality-blog", self.charm_state)
def process_kill(self, pid):
try:
os.kill(pid, 9)
except OSError:
pass
@inlineCallbacks
def get_agent_config(self):
# gets invoked by AgentTestBase.setUp
options = yield super(MachineAgentTest, self).get_agent_config()
machine_state_manager = MachineStateManager(self.client)
self.machine_state = yield machine_state_manager.add_machine_state()
self.change_environment(JUJU_MACHINE_ID="0",
JUJU_HOME=self.juju_directory)
options["machine_id"] = str(self.machine_state.id)
# Start the agent with watching enabled
returnValue(options)
@inlineCallbacks
def test_start_begins_watch_and_initializes_directories(self):
self.agent.set_watch_enabled(True)
mock_machine_state = self.mocker.patch(MachineState)
mock_machine_state.watch_assigned_units(self.agent.watch_service_units)
self.mocker.replay()
yield self.agent.startService()
self.assertTrue(os.path.isdir(self.agent.charms_directory))
self.assertTrue(os.path.isdir(self.agent.units_directory))
self.assertTrue(os.path.isdir(self.agent.unit_state_directory))
self.assertIn(
"Machine agent started id:%s" % self.agent.get_machine_id(),
self.output.getvalue())
yield self.agent.stopService()
def test_agent_machine_id_environment_extraction(self):
self.change_args("es-agent")
parser = argparse.ArgumentParser()
self.agent.setup_options(parser)
config = parser.parse_args(namespace=TwistedOptionNamespace())
self.assertEqual(config["machine_id"], "0")
def test_get_agent_name(self):
self.assertEqual(self.agent.get_agent_name(), "Machine:0")
def test_agent_machine_id_cli_error(self):
"""
If the machine id can't be found, a detailed error message
is given.
"""
# initially setup by get_agent_config in setUp
self.change_environment(JUJU_MACHINE_ID="")
self.change_args("es-agent", "--zookeeper-servers",
get_test_zookeeper_address(), "--juju-directory",
self.makeDir())
parser = argparse.ArgumentParser()
self.agent.setup_options(parser)
options = parser.parse_args(namespace=TwistedOptionNamespace())
e = self.assertRaises(JujuError, self.agent.configure, options)
self.assertIn(("--machine-id must be provided in the command line,"
" or $JUJU_MACHINE_ID in the environment"), str(e))
def test_agent_machine_id_cli_extraction(self):
"""Command line passing of machine id works and has precedence
over environment arg passing."""
self.change_environment(JUJU_MACHINE_ID=str(21))
self.change_args("es-agent", "--machine-id", "0")
parser = argparse.ArgumentParser()
self.agent.setup_options(parser)
config = parser.parse_args(namespace=TwistedOptionNamespace())
self.assertEqual(config["machine_id"], "0")
def test_machine_agent_knows_its_machine_id(self):
self.assertEqual(self.agent.get_machine_id(), "0")
@inlineCallbacks
def test_charm_download(self):
"""
Downloading a charm should store the charm locally.
"""
yield self.agent.startService()
yield self.agent.download_charm(self.charm_state)
checksum = self.charm.get_sha256()
charm_id = local_charm_id(self.charm)
charm_key = under.quote("%s:%s" % (charm_id, checksum))
charm_path = os.path.join(self.agent.charms_directory, charm_key)
self.assertTrue(os.path.exists(charm_path))
bundle = CharmBundle(charm_path)
self.assertEquals(bundle.get_revision(), self.charm.get_revision())
self.assertEquals(bundle.get_sha256(), checksum)
self.assertIn("Downloading charm %s" % charm_id,
self.output.getvalue())
@inlineCallbacks
def test_watch_new_service_unit(self):
"""
Adding a new service unit is detected by the watch.
"""
self.agent.set_watch_enabled(True)
yield self.agent.startService()
mock_deployment = self.mocker.patch(self.agent.deploy_factory)
mock_deployment.start("0", get_test_zookeeper_address(), MATCH_BUNDLE)
test_deferred = Deferred()
def test_complete(machine_id, servers, bundle):
test_deferred.callback(True)
self.mocker.call(test_complete)
self.mocker.replay()
# Create a new service unit
self.service_unit = yield self.service.add_unit_state()
yield self.service_unit.assign_to_machine(self.machine_state)
yield test_deferred
self.assertIn("Units changed old:set([]) new:set(['fatality-blog/0'])",
self.output.getvalue())
@inlineCallbacks
def test_watch_new_service_unit_error(self):
"""
An error while starting a new service is logged
"""
# Inject an error into the service deployment
mock_agent = self.mocker.patch(self.agent)
mock_agent.start_service_unit("fatality-blog/0")
self.mocker.result(fail(SyntaxError("Bad")))
self.mocker.replay()
yield self.agent.watch_service_units(None, set(["fatality-blog/0"]))
self.assertIn("Starting service unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("Error starting unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("SyntaxError: Bad", self.output.getvalue())
@inlineCallbacks
def test_service_unit_removed(self):
"""
Service unit removed with manual invocation of watch_service_units.
"""
# Start the agent with watching enabled
self.agent.set_watch_enabled(True)
yield self.agent.startService()
test_deferred = Deferred()
mock_deployment = self.mocker.patch(self.agent.deploy_factory)
mock_deployment.start("0", get_test_zookeeper_address(), MATCH_BUNDLE)
self.mocker.result(succeed(True))
mock_deployment.destroy()
self.mocker.result(succeed(True))
def test_complete():
test_deferred.callback(True)
self.mocker.call(test_complete)
self.mocker.replay()
# Create a new service unit
self.service_unit = yield self.service.add_unit_state()
yield self.service_unit.assign_to_machine(self.machine_state)
# And now remove it
yield self.service_unit.unassign_from_machine()
yield test_deferred
@inlineCallbacks
def test_watch_removed_service_unit_error(self):
"""
An error while removing a service unit is logged
"""
mock_agent = self.mocker.patch(self.agent)
mock_agent.kill_service_unit("fatality-blog/0")
self.mocker.result(fail(OSError("Bad")))
self.mocker.replay()
yield self.agent.watch_service_units(set(["fatality-blog/0"]), set())
self.assertIn("Stopping service unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("Error stopping unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("OSError: Bad", self.output.getvalue())
class MachineAgentTest(AgentTestBase, RepositoryTestBase):
agent_class = MachineAgent
@inlineCallbacks
def setUp(self):
yield super(MachineAgentTest, self).setUp()
self.output = self.capture_logging(
"juju.agents.machine", level=logging.DEBUG)
config = self.get_test_environment_config()
environment = config.get_default()
# Store the environment to zookeeper
environment_state_manager = EnvironmentStateManager(self.client)
yield environment_state_manager.set_config_state(config, "myfirstenv")
# Load the environment with the charm state and charm binary
self.provider = environment.get_machine_provider()
self.storage = self.provider.get_file_storage()
self.charm = CharmDirectory(self.sample_dir1)
self.publisher = CharmPublisher(self.client, self.storage)
yield self.publisher.add_charm(local_charm_id(self.charm), self.charm)
charm_states = yield self.publisher.publish()
self.charm_state = charm_states[0]
# Create a service from the charm from which we can create units for
# the machine.
self.service_state_manager = ServiceStateManager(self.client)
self.service = yield self.service_state_manager.add_service_state(
"fatality-blog", self.charm_state)
def process_kill(self, pid):
try:
os.kill(pid, 9)
except OSError:
pass
@inlineCallbacks
def get_agent_config(self):
# gets invoked by AgentTestBase.setUp
options = yield super(MachineAgentTest, self).get_agent_config()
machine_state_manager = MachineStateManager(self.client)
self.machine_state = yield machine_state_manager.add_machine_state()
self.change_environment(
JUJU_MACHINE_ID="0",
JUJU_HOME=self.juju_directory)
options["machine_id"] = str(self.machine_state.id)
# Start the agent with watching enabled
returnValue(options)
@inlineCallbacks
def test_start_begins_watch_and_initializes_directories(self):
self.agent.set_watch_enabled(True)
mock_machine_state = self.mocker.patch(MachineState)
mock_machine_state.watch_assigned_units(
self.agent.watch_service_units)
self.mocker.replay()
yield self.agent.startService()
self.assertTrue(os.path.isdir(self.agent.charms_directory))
self.assertTrue(os.path.isdir(self.agent.units_directory))
self.assertTrue(os.path.isdir(self.agent.unit_state_directory))
self.assertIn(
"Machine agent started id:%s" % self.agent.get_machine_id(),
self.output.getvalue())
yield self.agent.stopService()
def test_agent_machine_id_environment_extraction(self):
self.change_args("es-agent")
parser = argparse.ArgumentParser()
self.agent.setup_options(parser)
config = parser.parse_args(namespace=TwistedOptionNamespace())
self.assertEqual(
config["machine_id"], "0")
def test_get_agent_name(self):
self.assertEqual(self.agent.get_agent_name(), "Machine:0")
def test_agent_machine_id_cli_error(self):
"""
If the machine id can't be found, a detailed error message
is given.
"""
# initially setup by get_agent_config in setUp
self.change_environment(JUJU_MACHINE_ID="")
self.change_args("es-agent",
"--zookeeper-servers",
get_test_zookeeper_address(),
"--juju-directory",
self.makeDir())
parser = argparse.ArgumentParser()
self.agent.setup_options(parser)
options = parser.parse_args(namespace=TwistedOptionNamespace())
e = self.assertRaises(
JujuError,
self.agent.configure,
options)
self.assertIn(
("--machine-id must be provided in the command line,"
" or $JUJU_MACHINE_ID in the environment"),
str(e))
def test_agent_machine_id_cli_extraction(self):
"""Command line passing of machine id works and has precedence
over environment arg passing."""
self.change_environment(JUJU_MACHINE_ID=str(21))
self.change_args("es-agent", "--machine-id", "0")
parser = argparse.ArgumentParser()
self.agent.setup_options(parser)
config = parser.parse_args(namespace=TwistedOptionNamespace())
self.assertEqual(
config["machine_id"], "0")
def test_machine_agent_knows_its_machine_id(self):
self.assertEqual(self.agent.get_machine_id(), "0")
@inlineCallbacks
def test_charm_download(self):
"""
Downloading a charm should store the charm locally.
"""
yield self.agent.startService()
yield self.agent.download_charm(self.charm_state)
checksum = self.charm.get_sha256()
charm_id = local_charm_id(self.charm)
charm_key = under.quote("%s:%s" % (charm_id, checksum))
charm_path = os.path.join(self.agent.charms_directory, charm_key)
self.assertTrue(os.path.exists(charm_path))
bundle = CharmBundle(charm_path)
self.assertEquals(
bundle.get_revision(), self.charm.get_revision())
self.assertEquals(bundle.get_sha256(), checksum)
self.assertIn(
"Downloading charm %s" % charm_id, self.output.getvalue())
@inlineCallbacks
def test_watch_new_service_unit(self):
"""
Adding a new service unit is detected by the watch.
"""
self.agent.set_watch_enabled(True)
yield self.agent.startService()
mock_deployment = self.mocker.patch(self.agent.deploy_factory)
mock_deployment.start("0", get_test_zookeeper_address(), MATCH_BUNDLE)
test_deferred = Deferred()
def test_complete(machine_id, servers, bundle):
test_deferred.callback(True)
self.mocker.call(test_complete)
self.mocker.replay()
# Create a new service unit
self.service_unit = yield self.service.add_unit_state()
yield self.service_unit.assign_to_machine(self.machine_state)
yield test_deferred
self.assertIn(
"Units changed old:set([]) new:set(['fatality-blog/0'])",
self.output.getvalue())
@inlineCallbacks
def test_watch_new_service_unit_error(self):
"""
An error while starting a new service is logged
"""
# Inject an error into the service deployment
mock_agent = self.mocker.patch(self.agent)
mock_agent.start_service_unit("fatality-blog/0")
self.mocker.result(fail(SyntaxError("Bad")))
self.mocker.replay()
yield self.agent.watch_service_units(None, set(["fatality-blog/0"]))
self.assertIn("Starting service unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("Error starting unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("SyntaxError: Bad", self.output.getvalue())
@inlineCallbacks
def test_service_unit_removed(self):
"""
Service unit removed with manual invocation of watch_service_units.
"""
# Start the agent with watching enabled
self.agent.set_watch_enabled(True)
yield self.agent.startService()
test_deferred = Deferred()
mock_deployment = self.mocker.patch(self.agent.deploy_factory)
mock_deployment.start("0", get_test_zookeeper_address(), MATCH_BUNDLE)
self.mocker.result(succeed(True))
mock_deployment.destroy()
self.mocker.result(succeed(True))
def test_complete():
test_deferred.callback(True)
self.mocker.call(test_complete)
self.mocker.replay()
# Create a new service unit
self.service_unit = yield self.service.add_unit_state()
yield self.service_unit.assign_to_machine(self.machine_state)
# And now remove it
yield self.service_unit.unassign_from_machine()
yield test_deferred
@inlineCallbacks
def test_watch_removed_service_unit_error(self):
"""
An error while removing a service unit is logged
"""
mock_agent = self.mocker.patch(self.agent)
mock_agent.kill_service_unit("fatality-blog/0")
self.mocker.result(fail(OSError("Bad")))
self.mocker.replay()
yield self.agent.watch_service_units(set(["fatality-blog/0"]), set())
self.assertIn("Stopping service unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("Error stopping unit: %s" % "fatality-blog/0",
self.output.getvalue())
self.assertIn("OSError: Bad", self.output.getvalue())
class RemoteRepositoryTest(RepositoryTestBase):
def setUp(self):
super(RemoteRepositoryTest, self).setUp()
self.cache_path = os.path.join(
self.makeDir(), "notexistyet")
self.download_path = os.path.join(self.cache_path, "downloads")
def delete():
if os.path.exists(self.cache_path):
shutil.rmtree(self.cache_path)
self.addCleanup(delete)
self.charm = CharmDirectory(
os.path.join(self.unbundled_repo_path, "series", "dummy"))
with open(self.charm.as_bundle().path, "rb") as f:
self.bundle_data = f.read()
self.sha256 = self.charm.as_bundle().get_sha256()
self.getPage = self.mocker.replace("twisted.web.client.getPage")
self.downloadPage = self.mocker.replace(
"twisted.web.client.downloadPage")
def repo(self, url_base):
return RemoteCharmRepository(url_base, self.cache_path)
def cache_location(self, url_str, revision):
charm_url = CharmURL.parse(url_str)
cache_key = under.quote(
"%s.charm" % (charm_url.with_revision(revision)))
return os.path.join(self.cache_path, cache_key)
def charm_info(self, url_str, revision, warnings=None, errors=None):
info = {"revision": revision, "sha256": self.sha256}
if errors:
info["errors"] = errors
if warnings:
info["warnings"] = warnings
return json.dumps({url_str: info})
def mock_charm_info(self, url, result):
self.getPage(url)
self.mocker.result(result)
def mock_download(self, url, error=None):
self.downloadPage(url, ANY)
if error:
return self.mocker.result(fail(error))
def download(_, path):
self.assertTrue(path.startswith(self.download_path))
with open(path, "wb") as f:
f.write(self.bundle_data)
return succeed(None)
self.mocker.call(download)
@inlineCallbacks
def assert_find_uncached(self, dns_name, url_str, info_url, find_url):
self.mock_charm_info(info_url, succeed(self.charm_info(url_str, 1)))
self.mock_download(find_url)
self.mocker.replay()
repo = self.repo(dns_name)
charm = yield repo.find(CharmURL.parse(url_str))
self.assertEquals(charm.get_sha256(), self.sha256)
self.assertEquals(charm.path, self.cache_location(url_str, 1))
self.assertEquals(os.listdir(self.download_path), [])
@inlineCallbacks
def assert_find_cached(self, dns_name, url_str, info_url):
os.makedirs(self.cache_path)
cache_location = self.cache_location(url_str, 1)
shutil.copy(self.charm.as_bundle().path, cache_location)
self.mock_charm_info(info_url, succeed(self.charm_info(url_str, 1)))
self.mocker.replay()
repo = self.repo(dns_name)
charm = yield repo.find(CharmURL.parse(url_str))
self.assertEquals(charm.get_sha256(), self.sha256)
self.assertEquals(charm.path, cache_location)
def assert_find_error(self, dns_name, url_str, err_type, message):
self.mocker.replay()
repo = self.repo(dns_name)
d = self.assertFailure(repo.find(CharmURL.parse(url_str)), err_type)
def verify(error):
self.assertEquals(str(error), message)
d.addCallback(verify)
return d
@inlineCallbacks
def assert_latest(self, dns_name, url_str, revision):
self.mocker.replay()
repo = self.repo(dns_name)
result = yield repo.latest(CharmURL.parse(url_str))
self.assertEquals(result, revision)
def assert_latest_error(self, dns_name, url_str, err_type, message):
self.mocker.replay()
repo = self.repo(dns_name)
d = self.assertFailure(repo.latest(CharmURL.parse(url_str)), err_type)
def verify(error):
self.assertEquals(str(error), message)
d.addCallback(verify)
return d
def test_find_plain_uncached(self):
return self.assert_find_uncached(
"https://somewhe.re", "cs:series/name",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name",
"https://somewhe.re/charm/series/name-1")
def test_find_revision_uncached(self):
return self.assert_find_uncached(
"https://somewhe.re", "cs:series/name-1",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name-1",
"https://somewhe.re/charm/series/name-1")
def test_find_user_uncached(self):
return self.assert_find_uncached(
"https://somewhereel.se", "cs:~user/srs/name",
"https://somewhereel.se/charm-info?charms=cs%3A%7Euser/srs/name",
"https://somewhereel.se/charm/%7Euser/srs/name-1")
def test_find_plain_cached(self):
return self.assert_find_cached(
"https://somewhe.re", "cs:series/name",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name")
def test_find_revision_cached(self):
return self.assert_find_cached(
"https://somewhe.re", "cs:series/name-1",
"https://somewhe.re/charm-info?charms=cs%3Aseries/name-1")
def test_find_user_cached(self):
return self.assert_find_cached(
"https://somewhereel.se", "cs:~user/srs/name",
"https://somewhereel.se/charm-info?charms=cs%3A%7Euser/srs/name")
def test_find_info_http_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
fail(Error("500")))
return self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmNotFound,
"Charm 'cs:series/name' not found in repository https://anoth.er")
@inlineCallbacks
def test_find_info_store_warning(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name-1",
succeed(self.charm_info(
"cs:series/name-1", 1, warnings=["omg", "halp"])))
self.mock_download("https://anoth.er/charm/series/name-1")
self.mocker.replay()
repo = self.repo("https://anoth.er")
log = self.capture_logging("juju.charm")
charm = yield repo.find(CharmURL.parse("cs:series/name-1"))
self.assertIn("omg", log.getvalue())
self.assertIn("halp", log.getvalue())
self.assertEquals(charm.get_sha256(), self.sha256)
def test_find_info_store_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name-101",
succeed(self.charm_info(
"cs:series/name-101", 101, errors=["oh", "noes"])))
return self.assert_find_error(
"https://anoth.er", "cs:series/name-101", CharmError,
"Error processing 'cs:series/name-101': oh; noes")
def test_find_info_bad_revision(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name-99",
succeed(self.charm_info("cs:series/name-99", 1)))
return self.assert_find_error(
"https://anoth.er", "cs:series/name-99", AssertionError,
"bad url revision")
def test_find_download_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(json.dumps({"cs:series/name": {"revision": 123}})))
self.mock_download(
"https://anoth.er/charm/series/name-123", Error("999"))
return self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmNotFound,
"Charm 'cs:series/name-123' not found in repository "
"https://anoth.er")
def test_find_charm_revision_mismatch(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(json.dumps({"cs:series/name": {"revision": 99}})))
self.mock_download("https://anoth.er/charm/series/name-99")
return self.assert_find_error(
"https://anoth.er", "cs:series/name", AssertionError,
"bad charm revision")
@inlineCallbacks
def test_find_downloaded_hash_mismatch(self):
cache_location = self.cache_location("cs:series/name-1", 1)
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(json.dumps(
{"cs:series/name": {"revision": 1, "sha256": "NO YUO"}})))
self.mock_download("https://anoth.er/charm/series/name-1")
yield self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmError,
"Error processing 'cs:series/name-1 (downloaded)': SHA256 "
"mismatch")
self.assertFalse(os.path.exists(cache_location))
@inlineCallbacks
def test_find_cached_hash_mismatch(self):
os.makedirs(self.cache_path)
cache_location = self.cache_location("cs:series/name-1", 1)
shutil.copy(self.charm.as_bundle().path, cache_location)
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(json.dumps(
{"cs:series/name": {"revision": 1, "sha256": "NO YUO"}})))
yield self.assert_find_error(
"https://anoth.er", "cs:series/name", CharmError,
"Error processing 'cs:series/name-1 (cached)': SHA256 mismatch")
self.assertFalse(os.path.exists(cache_location))
def test_latest_plain(self):
self.mock_charm_info(
"https://somewhe.re/charm-info?charms=cs%3Afoo/bar",
succeed(self.charm_info("cs:foo/bar", 99)))
return self.assert_latest("https://somewhe.re", "cs:foo/bar-1", 99)
def test_latest_user(self):
self.mock_charm_info(
"https://somewhereel.se/charm-info?charms=cs%3A%7Efee/foo/bar",
succeed(self.charm_info("cs:~fee/foo/bar", 123)))
return self.assert_latest(
"https://somewhereel.se", "cs:~fee/foo/bar", 123)
def test_latest_revision(self):
self.mock_charm_info(
"https://somewhereel.se/charm-info?charms=cs%3A%7Efee/foo/bar",
succeed(self.charm_info("cs:~fee/foo/bar", 123)))
return self.assert_latest(
"https://somewhereel.se", "cs:~fee/foo/bar-99", 123)
def test_latest_http_error(self):
self.mock_charm_info(
"https://andanoth.er/charm-info?charms=cs%3A%7Eblib/blab/blob",
fail(Error("404")))
return self.assert_latest_error(
"https://andanoth.er", "cs:~blib/blab/blob", CharmNotFound,
"Charm 'cs:~blib/blab/blob' not found in repository "
"https://andanoth.er")
@inlineCallbacks
def test_latest_store_warning(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(self.charm_info(
"cs:series/name", 1, warnings=["eww", "yuck"])))
self.mocker.replay()
repo = self.repo("https://anoth.er")
log = self.capture_logging("juju.charm")
revision = yield repo.latest(CharmURL.parse("cs:series/name-1"))
self.assertIn("eww", log.getvalue())
self.assertIn("yuck", log.getvalue())
self.assertEquals(revision, 1)
def test_latest_store_error(self):
self.mock_charm_info(
"https://anoth.er/charm-info?charms=cs%3Aseries/name",
succeed(self.charm_info(
"cs:series/name", 1, errors=["blam", "dink"])))
return self.assert_latest_error(
"https://anoth.er", "cs:series/name-1", CharmError,
"Error processing 'cs:series/name': blam; dink")
def test_repo_type(self):
self.mocker.replay()
self.assertEqual(self.repo("http://fbaro.com").type, "store")
def test_config(self):
"""Validate that ConfigOptions are available on the charm"""
from juju.charm.tests.test_config import sample_yaml_data
directory = CharmDirectory(sample_directory)
self.assertEquals(directory.config.get_serialization_data(),
sample_yaml_data)
def test_as_directory(self):
directory = CharmDirectory(self.sample_dir1)
self.assertIs(directory.as_directory(), directory)
def test_as_directory(self):
directory = CharmDirectory(self.sample_dir1)
self.assertIs(directory.as_directory(), directory)