Exemple #1
0
def download_charm(client, charm_id, charms_directory):
    """Retrieve a charm from the provider storage to the local machine.
    """
    charm_state_manager = CharmStateManager(client)
    charm_state = yield charm_state_manager.get_charm_state(charm_id)

    # Calculate local charm path
    checksum = yield charm_state.get_sha256()
    charm_key = under.quote("%s:%s" % (charm_state.id, checksum))
    local_charm_path = os.path.join(
        charms_directory, charm_key)

    # Retrieve charm from provider storage link
    if charm_state.bundle_url.startswith("file://"):
        file_path = charm_state.bundle_url[len("file://"):]
        if not os.path.exists(file_path):
            raise FileNotFound(charm_state.bundle_url)
        shutil.copyfileobj(open(file_path), open(local_charm_path, "w"))
    else:
        try:
            yield downloadPage(charm_state.bundle_url, local_charm_path)
        except Error:
            raise FileNotFound(charm_state.bundle_url)

    returnValue(CharmBundle(local_charm_path))
Exemple #2
0
    def setUp(self):
        super(CharmPublisherTest, self).setUp()
        zookeeper.set_debug_level(0)

        self.charm = CharmDirectory(self.sample_dir1)
        self.charm_id = local_charm_id(self.charm)
        self.charm_key = under.quote(self.charm_id)
        # provider storage key
        self.charm_storage_key = under.quote(
            "%s:%s" % (self.charm_id, self.charm.get_sha256()))

        self.client = ZookeeperClient(get_test_zookeeper_address())
        self.storage_dir = self.makeDir()
        self.storage = FileStorage(self.storage_dir)
        self.publisher = CharmPublisher(self.client, self.storage)

        yield self.client.connect()
        yield self.client.create("/charms")
Exemple #3
0
    def setUp(self):
        super(CharmPublisherTest, self).setUp()
        zookeeper.set_debug_level(0)

        self.charm = CharmDirectory(self.sample_dir1)
        self.charm_id = local_charm_id(self.charm)
        self.charm_key = under.quote(self.charm_id)
        # provider storage key
        self.charm_storage_key = under.quote(
            "%s:%s" % (self.charm_id, self.charm.get_sha256()))

        self.client = ZookeeperClient(get_test_zookeeper_address())
        self.storage_dir = self.makeDir()
        self.storage = FileStorage(self.storage_dir)
        self.publisher = CharmPublisher(self.client, self.storage)

        yield self.client.connect()
        yield self.client.create("/charms")
Exemple #4
0
    def test_add_charm_with_concurrent(self):
        """
        Publishing a charm, that has become published concurrent, after the
        add_charm, works fine. it will write to storage regardless. The use
        of a sha256 as part of the storage key is utilized to help ensure
        uniqueness of bits. The sha256 is also stored with the charm state.

        This relation betewen the charm state and the binary bits, helps
        guarantee the property that any published charm in zookeeper will use
        the binary bits that it was published with.
        """

        yield self.publisher.add_charm(self.charm_id, self.charm)

        concurrent_publisher = CharmPublisher(self.client, self.storage)

        charm = CharmDirectory(self.sample_dir1)
        yield concurrent_publisher.add_charm(self.charm_id, charm)

        yield self.publisher.publish()

        # modify the charm to create a conflict scenario
        self.makeFile("zebra", path=os.path.join(self.sample_dir1, "junk.txt"))

        # assert the charm now has a different sha post modification
        modified_charm_sha = charm.get_sha256()
        self.assertNotEqual(modified_charm_sha, self.charm.get_sha256())

        # verify publishing raises a stateerror
        def verify_failure(result):
            if not isinstance(result, Failure):
                self.fail("Should have raised state error")
            result.trap(StateChanged)
            return True

        yield concurrent_publisher.publish().addBoth(verify_failure)

        # verify the zk state
        charm_nodes = yield self.client.get_children("/charms")
        self.assertEqual(charm_nodes, [self.charm_key])

        content, stat = yield self.client.get("/charms/%s" % charm_nodes[0])

        # assert the checksum matches the initially published checksum
        self.assertEqual(yaml.load(content)["sha256"], self.charm.get_sha256())

        store_path = os.path.join(self.storage_dir, self.charm_storage_key)
        self.assertTrue(os.path.exists(store_path))

        # and the binary bits where stored
        modified_charm_storage_key = under.quote(
            "%s:%s" % (self.charm_id, modified_charm_sha))
        modified_store_path = os.path.join(self.storage_dir,
                                           modified_charm_storage_key)
        self.assertTrue(os.path.exists(modified_store_path))
Exemple #5
0
    def test_charm_download(self):
        """Downloading a charm should store the charm locally."""
        yield self.unit_manager.download_charm(self.charm_state)
        checksum = self.charm.get_sha256()
        charm_id = local_charm_id(self.charm)
        charm_key = under.quote("%s:%s" % (charm_id, checksum))
        charm_path = os.path.join(self.unit_manager.charms_directory, charm_key)

        self.assertTrue(os.path.exists(charm_path))
        bundle = CharmBundle(charm_path)
        self.assertEquals(
            bundle.get_revision(), self.charm.get_revision())
        self.assertEquals(bundle.get_sha256(), checksum)
        self.assertIn(
            "Downloading charm %s" % charm_id, self.output.getvalue())
Exemple #6
0
    def test_charm_download(self):
        """
        Downloading a charm should store the charm locally.
        """
        yield self.agent.startService()
        yield self.agent.download_charm(self.charm_state)

        checksum = self.charm.get_sha256()
        charm_id = local_charm_id(self.charm)
        charm_key = under.quote("%s:%s" % (charm_id, checksum))
        charm_path = os.path.join(self.agent.charms_directory, charm_key)

        self.assertTrue(os.path.exists(charm_path))
        bundle = CharmBundle(charm_path)
        self.assertEquals(bundle.get_revision(), self.charm.get_revision())
        self.assertEquals(bundle.get_sha256(), checksum)
        self.assertIn("Downloading charm %s" % charm_id,
                      self.output.getvalue())
Exemple #7
0
    def test_charm_download_file(self):
        """Downloading a charm should store the charm locally.
        """
        charm, charm_state = yield self.publish_charm()
        charm_directory = self.makeDir()

        # Download the charm
        yield download_charm(self.client, charm_state.id, charm_directory)

        # Verify the downloaded copy
        checksum = charm.get_sha256()
        charm_id = local_charm_id(charm)
        charm_key = under.quote("%s:%s" % (charm_id, checksum))
        charm_path = os.path.join(charm_directory, charm_key)

        self.assertTrue(os.path.exists(charm_path))
        bundle = CharmBundle(charm_path)
        self.assertEquals(bundle.get_revision(), charm.get_revision())

        self.assertEqual(checksum, bundle.get_sha256())
Exemple #8
0
def download_charm(client, charm_id, charms_directory):
    charm_state_manager = CharmStateManager(client)
    charm_state = yield charm_state_manager.get_charm_state(charm_id)

    # Calculate local charm path
    checksum = yield charm_state.get_sha256()
    charm_key = under.quote("%s:%s" % (charm_state.id, checksum))
    local_charm_path = os.path.join(charms_directory, charm_key)

    # Retrieve charm from provider storage link
    if charm_state.bundle_url.startswith("file://"):
        file_path = charm_state.bundle_url[len("file://"):]
        if not os.path.exists(file_path):
            raise FileNotFound(charm_state.bundle_url)
        shutil.copyfileobj(open(file_path), open(local_charm_path, "w"))
    else:
        try:
            yield downloadPage(charm_state.bundle_url, local_charm_path)
        except Error:
            raise FileNotFound(charm_state.bundle_url)
    returnValue(CharmBundle(local_charm_path))
Exemple #9
0
    def test_charm_download_file(self):
        """Downloading a charm should store the charm locally.
        """
        charm, charm_state = yield self.publish_charm()
        charm_directory = self.makeDir()

        # Download the charm
        yield download_charm(
            self.client, charm_state.id, charm_directory)

        # Verify the downloaded copy
        checksum = charm.get_sha256()
        charm_id = local_charm_id(charm)
        charm_key = under.quote("%s:%s" % (charm_id, checksum))
        charm_path = os.path.join(charm_directory, charm_key)

        self.assertTrue(os.path.exists(charm_path))
        bundle = CharmBundle(charm_path)
        self.assertEquals(bundle.get_revision(), charm.get_revision())

        self.assertEqual(checksum, bundle.get_sha256())
Exemple #10
0
    def _publish_one(self, charm_id, charm):
        if charm_id in self._charm_state_cache:
            return succeed(self._charm_state_cache[charm_id])

        bundle = charm.as_bundle()
        charm_file = open(bundle.path, "rb")
        charm_store_path = under.quote("%s:%s" %
                                       (charm_id, bundle.get_sha256()))

        def close_charm_file(passthrough):
            charm_file.close()
            return passthrough

        def get_charm_url(result):
            return self._storage.get_url(charm_store_path)

        d = self._storage.put(charm_store_path, charm_file)
        d.addBoth(close_charm_file)
        d.addCallback(get_charm_url)
        d.addCallback(self._cb_store_charm_state, charm_id, bundle)
        d.addErrback(self._eb_verify_duplicate, charm_id, bundle)
        return d
Exemple #11
0
    def _publish_one(self, charm_id, charm):
        if charm_id in self._charm_state_cache:
            return succeed(self._charm_state_cache[charm_id])

        bundle = charm.as_bundle()
        charm_file = open(bundle.path, "rb")
        charm_store_path = under.quote(
            "%s:%s" % (charm_id, bundle.get_sha256()))

        def close_charm_file(passthrough):
            charm_file.close()
            return passthrough

        def get_charm_url(result):
            return self._storage.get_url(charm_store_path)

        d = self._storage.put(charm_store_path, charm_file)
        d.addBoth(close_charm_file)
        d.addCallback(get_charm_url)
        d.addCallback(self._cb_store_charm_state, charm_id, bundle)
        d.addErrback(self._eb_verify_duplicate, charm_id, bundle)
        return d
Exemple #12
0
    def test_add_charm_with_concurrent(self):
        """
        Publishing a charm, that has become published concurrent, after the
        add_charm, works fine. it will write to storage regardless. The use
        of a sha256 as part of the storage key is utilized to help ensure
        uniqueness of bits. The sha256 is also stored with the charm state.

        This relation betewen the charm state and the binary bits, helps
        guarantee the property that any published charm in zookeeper will use
        the binary bits that it was published with.
        """

        yield self.publisher.add_charm(self.charm_id, self.charm)

        concurrent_publisher = CharmPublisher(
            self.client, self.storage)

        charm = CharmDirectory(self.sample_dir1)
        yield concurrent_publisher.add_charm(self.charm_id, charm)

        yield self.publisher.publish()

        # modify the charm to create a conflict scenario
        self.makeFile("zebra",
                      path=os.path.join(self.sample_dir1, "junk.txt"))

        # assert the charm now has a different sha post modification
        modified_charm_sha = charm.get_sha256()
        self.assertNotEqual(
            modified_charm_sha,
            self.charm.get_sha256())

        # verify publishing raises a stateerror
        def verify_failure(result):
            if not isinstance(result, Failure):
                self.fail("Should have raised state error")
            result.trap(StateChanged)
            return True

        yield concurrent_publisher.publish().addBoth(verify_failure)

        # verify the zk state
        charm_nodes = yield self.client.get_children("/charms")
        self.assertEqual(charm_nodes, [self.charm_key])

        content, stat = yield self.client.get(
            "/charms/%s" % charm_nodes[0])

        # assert the checksum matches the initially published checksum
        self.assertEqual(
            yaml.load(content)["sha256"], self.charm.get_sha256())

        store_path = os.path.join(self.storage_dir, self.charm_storage_key)
        self.assertTrue(os.path.exists(store_path))

        # and the binary bits where stored
        modified_charm_storage_key = under.quote(
            "%s:%s" % (self.charm_id, modified_charm_sha))
        modified_store_path = os.path.join(
            self.storage_dir, modified_charm_storage_key)
        self.assertTrue(os.path.exists(modified_store_path))
Exemple #13
0
 def cache_location(self, url_str, revision):
     charm_url = CharmURL.parse(url_str)
     cache_key = under.quote("%s.charm" %
                             (charm_url.with_revision(revision)))
     return os.path.join(self.cache_path, cache_key)
Exemple #14
0
def _cache_key(charm_url):
    charm_url.assert_revision()
    return under.quote("%s.charm" % charm_url)
Exemple #15
0
def _charm_path(charm_id):
    return "/charms/%s" % under.quote(charm_id)
Exemple #16
0
def _cache_key(charm_url):
    charm_url.assert_revision()
    return under.quote("%s.charm" % charm_url)
Exemple #17
0
 def test_unmodified(self):
     s = string.ascii_letters + string.digits + "-."
     q = quote(s)
     self.assertEquals(quote(s), s)
     self.assertTrue(isinstance(q, str))
Exemple #18
0
 def test_coincidentally_unicode(self):
     s = u"hello_there/how'are~you-today.sir"
     q = quote(s)
     self.assertEquals(q, "hello_5f_there_2f_how_27_are_7e_you-today.sir")
     self.assertTrue(isinstance(q, str))
Exemple #19
0
def _charm_path(charm_id):
    return "/charms/%s" % under.quote(charm_id)
 def cache_location(self, url_str, revision):
     charm_url = CharmURL.parse(url_str)
     cache_key = under.quote(
         "%s.charm" % (charm_url.with_revision(revision)))
     return os.path.join(self.cache_path, cache_key)