Ejemplo n.º 1
0
class OpenStackUpgrader(UpgradeEngine):
    """OpenStack Upgrader.

    The class is designed to do the following tasks:

    * install manifests in the system
    * add new releases to nailgun's database
    * add notification about new releases
    """

    def __init__(self, *args, **kwargs):
        super(OpenStackUpgrader, self).__init__(*args, **kwargs)

        #: a list of releases to install
        self.releases = self._read_releases()
        #: a nailgun object - api wrapper
        self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])

        self._reset_state()

    def upgrade(self):
        self._reset_state()

        self.install_puppets()
        self.install_releases()
        self.install_versions()

    def rollback(self):
        self.remove_releases()
        self.remove_puppets()
        self.remove_versions()

    def install_puppets(self):
        logger.info('Installing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.copy(source, destination)

    def remove_puppets(self):
        logger.info('Removing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.remove(destination)

    def install_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        utils.create_dir_if_not_exists(release_versions_cfg['dst'])
        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.copy(version_file, dst)

    def remove_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.remove(dst)

    def install_releases(self):
        # add only new releases to nailgun and inject paths to
        # base repo if needed
        existing_releases = self.nailgun.get_releases()
        releases = self._get_unique_releases(self.releases, existing_releases)

        # upload unexisting releases
        for release in releases:
            # register new release
            logger.debug('Register a new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_release(release)
            # save release id for futher possible rollback
            self._rollback_ids['release'].append(response['id'])
            self.upload_release_deployment_tasks(response)

            if not release.get('is_deployable', True):
                continue

            # add notification abot successfull releases
            logger.debug('Add notification about new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_notification({
                'topic': 'release',
                'message': 'New release available: {0} ({1})'.format(
                    release['name'],
                    release['version'],
                ),
            })
            # save notification id for futher possible rollback
            self._rollback_ids['notification'].append(response['id'])

    def upload_release_deployment_tasks(self, release):
        """Performs os.walk by puppet src, matches all files with tasks
        of given pattern and uploads this for release.

        :param release: dict representation of release
        """
        tasks = []
        release_puppet_path = os.path.join(
            self.config.openstack['puppets']['dst'], release['version'])

        for file_path in utils.iterfiles_filter(
                release_puppet_path,
                self.config.deployment_tasks_file_pattern):

            tasks.extend(utils.read_from_yaml(file_path))

        self.nailgun.put_deployment_tasks(release, tasks)

    def remove_releases(self):
        """Remove all releases that are created by current session.
        """
        for release_id in reversed(self._rollback_ids['release']):
            try:
                logger.debug('Removing release with ID=%s', release_id)
                self.nailgun.remove_release(release_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception('%s', six.text_type(exc))

        for notif_id in reversed(self._rollback_ids['notification']):
            try:
                logger.debug('Removing notification with ID=%s', notif_id)
                self.nailgun.remove_notification(notif_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception('%s', six.text_type(exc))

    def _reset_state(self):
        """Remove rollback IDs from the arrays.
        """
        #: a list of ids that have to be removed in case of rollback
        self._rollback_ids = {
            'release': [],
            'notification': [],
        }

    @classmethod
    def _get_unique_releases(cls, releases, existing_releases):
        """Returns a list of releases that aren't exist yet.

        :param releases: a list of releases to filter
        :param existing_releases: a list of existing releases
        :returns: a list of unique releases
        """
        existing_releases = [
            (r['name'], r['version']) for r in existing_releases
        ]

        unique = lambda r: (r['name'], r['version']) not in existing_releases
        return [r for r in releases if unique(r)]

    def _read_releases(self):
        """Returns a list of releases in a dict representation.
        """
        releases = []

        # read releases from a set of files
        for release_yaml in glob.glob(self.config.openstack['releases']):
            with io.open(release_yaml, 'r', encoding='utf-8') as f:
                releases.extend(utils.load_fixture(f))

        return releases

    @property
    def required_free_space(self):
        spaces = {
            self.config.openstack['puppets']['dst']:
            glob.glob(self.config.openstack['puppets']['src']), }

        for dst, srcs in six.iteritems(spaces):
            size = 0
            for src in srcs:
                size += utils.dir_size(src)
            spaces[dst] = size

        return spaces
Ejemplo n.º 2
0
class TestNailgunClient(base.BaseTestCase):
    def setUp(self):
        mock_keystone = mock.MagicMock()
        self.mock_request = mock_keystone.request
        with mock.patch('fuel_upgrade.clients.nailgun_client.KeystoneClient',
                        return_value=mock_keystone):
            self.nailgun = NailgunClient('127.0.0.1', 8000)

    def test_create_release(self):
        # test normal bahavior
        self.mock_request.post.return_value = self.mock_requests_response(
            201, '{ "id": "42" }')

        response = self.nailgun.create_release(
            {'name': 'Havana on Ubuntu 12.04'})

        self.assertEqual(response, {'id': '42'})

        # test failed result
        self.mock_request.post.return_value.status_code = 409
        self.assertRaises(requests.exceptions.HTTPError,
                          self.nailgun.create_release,
                          {'name': 'Havana on Ubuntu 12.04'})

    def test_delete_release(self):
        # test normal bahavior
        for status in (200, 204):
            self.mock_request.delete.return_value = \
                self.mock_requests_response(status, 'No Content')
            response = self.nailgun.remove_release(42)
            self.assertEqual(response, 'No Content')

        # test failed result
        self.mock_request.delete.return_value = self.mock_requests_response(
            409, 'Conflict')

        self.assertRaises(requests.exceptions.HTTPError,
                          self.nailgun.remove_release, 42)

    def test_create_notification(self):
        # test normal bahavior
        self.mock_request.post.return_value = self.mock_requests_response(
            201, '{ "id": "42" }')

        response = self.nailgun.create_notification({
            'topic':
            'release',
            'message':
            'New release available!'
        })

        self.assertEqual(response, {'id': '42'})

        # test failed result
        self.mock_request.post.return_value.status_code = 409
        self.assertRaises(requests.exceptions.HTTPError,
                          self.nailgun.create_notification, {
                              'topic': 'release',
                              'message': 'New release available!'
                          })

    def test_delete_notification(self):
        # test normal bahavior
        for status in (200, 204):
            self.mock_request.delete.return_value = \
                self.mock_requests_response(status, 'No Content')
            response = self.nailgun.remove_notification(42)
            self.assertEqual(response, 'No Content')

        # test failed result
        self.mock_request.delete.return_value = self.mock_requests_response(
            409, 'Conflict')

        self.assertRaises(requests.exceptions.HTTPError,
                          self.nailgun.remove_notification, 42)

    def test_get_tasks(self):
        # test positive cases
        self.mock_request.get.return_value = self.mock_requests_response(
            200, '[1,2,3]')
        response = self.nailgun.get_tasks()
        self.assertEqual(response, [1, 2, 3])

        # test negative cases
        self.mock_request.get.return_value = self.mock_requests_response(
            502, 'Bad gateway')

        self.assertRaises(requests.exceptions.HTTPError,
                          self.nailgun.get_tasks)
Ejemplo n.º 3
0
class OpenStackUpgrader(UpgradeEngine):
    """OpenStack Upgrader.

    The class is designed to do the following tasks:

    * install repos in the system
    * install manifests in the system
    * add new releases to nailgun's database
    * add notification about new releases
    """

    def __init__(self, *args, **kwargs):
        super(OpenStackUpgrader, self).__init__(*args, **kwargs)

        #: a list of releases to install
        self.releases = self._read_releases()
        #: a nailgun object - api wrapper
        self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])

        self._reset_state()

    def upgrade(self):
        self._reset_state()

        self.install_puppets()
        self.install_repos()
        self.install_releases()
        self.install_versions()

    def rollback(self):
        self.remove_releases()
        self.remove_repos()
        self.remove_puppets()
        self.remove_versions()

    def install_puppets(self):
        logger.info('Installing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.copy(source, destination)

    def remove_puppets(self):
        logger.info('Removing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.remove(destination)

    def install_repos(self):
        logger.info('Installing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['repos']['dst'],
                os.path.basename(source))
            utils.copy(source, destination)

    def remove_repos(self):
        logger.info('Removing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['repos']['dst'],
                os.path.basename(source))
            utils.remove(destination)

    def on_success(self):
        """Do nothing for this engine
        """

    def install_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        utils.create_dir_if_not_exists(release_versions_cfg['dst'])
        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.copy(version_file, dst)

    def remove_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.remove(dst)

    def install_releases(self):
        # check releases for existing in nailgun side
        releases = self._get_unique_releases(
            self.releases, self.nailgun.get_releases())

        # upload unexisting releases
        for release in releases:
            # register new release
            logger.debug('Register a new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_release(release)
            # save release id for futher possible rollback
            self._rollback_ids['release'].append(response['id'])

            # add notification abot successfull releases
            logger.debug('Add notification about new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_notification({
                'topic': 'release',
                'message': 'New release available: {0} ({1})'.format(
                    release['name'],
                    release['version'],
                ),
            })
            # save notification id for futher possible rollback
            self._rollback_ids['notification'].append(response['id'])

    def remove_releases(self):
        """Remove all releases that are created by current session.
        """
        for release_id in reversed(self._rollback_ids['release']):
            try:
                logger.debug('Removing release with ID=%s', release_id)
                self.nailgun.remove_release(release_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception(six.text_type(exc))

        for notif_id in reversed(self._rollback_ids['notification']):
            try:
                logger.debug('Removing notification with ID=%s', notif_id)
                self.nailgun.remove_notification(notif_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception(six.text_type(exc))

    def _reset_state(self):
        """Remove rollback IDs from the arrays.
        """
        #: a list of ids that have to be removed in case of rollback
        self._rollback_ids = {
            'release': [],
            'notification': [],
        }

    @classmethod
    def _get_unique_releases(cls, releases, existing_releases):
        """Returns a list of releases that aren't exist yet.

        :param releases: a list of releases to filter
        :param existing_releases: a list of existing releases
        :returns: a list of unique releases
        """
        existing_releases = [
            (r['name'], r['version']) for r in existing_releases
        ]

        unique = lambda r: (r['name'], r['version']) not in existing_releases
        return [r for r in releases if unique(r)]

    def _read_releases(self):
        """Returns a list of releases in a dict representation.
        """
        releases = []

        # read releases from a set of files
        for release_yaml in glob.glob(self.config.openstack['releases']):
            with io.open(release_yaml, 'r', encoding='utf-8') as f:
                releases.extend(utils.load_fixture(f))

        # inject orchestrator_data into releases if empty
        for release in releases:
            meta = {
                'version': release['version'],
                'master_ip': self.config.astute['ADMIN_NETWORK']['ipaddress']}

            if 'ubuntu' == release['operating_system'].lower():
                repo = 'http://{master_ip}:8080/{version}/ubuntu/x86_64 ' \
                       'precise main'
            else:
                repo = 'http://{master_ip}:8080/{version}/centos/x86_64'

            release['orchestrator_data'] = {
                'puppet_manifests_source': (
                    'rsync://{master_ip}:/puppet/{version}/manifests/'.format(
                        **meta)),
                'puppet_modules_source': (
                    'rsync://{master_ip}:/puppet/{version}/modules/'.format(
                        **meta)),
                'repo_metadata': {
                    'nailgun': repo.format(**meta)}}

        return releases

    @property
    def required_free_space(self):
        spaces = {
            self.config.openstack['puppets']['dst']:
            glob.glob(self.config.openstack['puppets']['src']),

            self.config.openstack['repos']['dst']:
            glob.glob(self.config.openstack['repos']['src'])}

        for dst, srcs in six.iteritems(spaces):
            size = 0
            for src in srcs:
                size += utils.dir_size(src)
            spaces[dst] = size

        return spaces
Ejemplo n.º 4
0
class OpenStackUpgrader(UpgradeEngine):
    """OpenStack Upgrader.

    The class is designed to do the following tasks:

    * install repos in the system
    * install manifests in the system
    * add new releases to nailgun's database
    * add notification about new releases
    """

    def __init__(self, *args, **kwargs):
        super(OpenStackUpgrader, self).__init__(*args, **kwargs)

        #: a list of releases to install
        self.releases = self._read_releases()
        #: a nailgun object - api wrapper
        self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])

        self._reset_state()

    def upgrade(self):
        self._reset_state()

        self.install_puppets()
        self.install_repos()
        self.install_releases()
        self.install_versions()

    def rollback(self):
        self.remove_releases()
        self.remove_repos()
        self.remove_puppets()
        self.remove_versions()

    def install_puppets(self):
        logger.info('Installing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.copy(source, destination)

    def remove_puppets(self):
        logger.info('Removing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.remove(destination)

    def install_repos(self):
        logger.info('Installing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['repos']['dst'],
                os.path.basename(source))
            utils.copy(source, destination)

    def remove_repos(self):
        logger.info('Removing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['repos']['dst'],
                os.path.basename(source))
            utils.remove(destination)

    def on_success(self):
        """Do nothing for this engine
        """

    def install_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        utils.create_dir_if_not_exists(release_versions_cfg['dst'])
        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.copy(version_file, dst)

    def remove_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.remove(dst)

    def install_releases(self):
        # add only new releases to nailgun and inject paths to
        # base repo if needed
        existing_releases = self.nailgun.get_releases()
        releases = self._get_unique_releases(self.releases, existing_releases)
        self._add_base_repos_to_releases(releases, existing_releases)

        # upload unexisting releases
        for release in releases:
            # register new release
            logger.debug('Register a new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_release(release)
            # save release id for futher possible rollback
            self._rollback_ids['release'].append(response['id'])

            # add notification abot successfull releases
            logger.debug('Add notification about new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_notification({
                'topic': 'release',
                'message': 'New release available: {0} ({1})'.format(
                    release['name'],
                    release['version'],
                ),
            })
            # save notification id for futher possible rollback
            self._rollback_ids['notification'].append(response['id'])

    def remove_releases(self):
        """Remove all releases that are created by current session.
        """
        for release_id in reversed(self._rollback_ids['release']):
            try:
                logger.debug('Removing release with ID=%s', release_id)
                self.nailgun.remove_release(release_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception('%s', six.text_type(exc))

        for notif_id in reversed(self._rollback_ids['notification']):
            try:
                logger.debug('Removing notification with ID=%s', notif_id)
                self.nailgun.remove_notification(notif_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception('%s', six.text_type(exc))

    def _reset_state(self):
        """Remove rollback IDs from the arrays.
        """
        #: a list of ids that have to be removed in case of rollback
        self._rollback_ids = {
            'release': [],
            'notification': [],
        }

    @classmethod
    def _get_unique_releases(cls, releases, existing_releases):
        """Returns a list of releases that aren't exist yet.

        :param releases: a list of releases to filter
        :param existing_releases: a list of existing releases
        :returns: a list of unique releases
        """
        existing_releases = [
            (r['name'], r['version']) for r in existing_releases
        ]

        unique = lambda r: (r['name'], r['version']) not in existing_releases
        return [r for r in releases if unique(r)]

    def _read_releases(self):
        """Returns a list of releases in a dict representation.
        """
        releases = []

        # read releases from a set of files
        for release_yaml in glob.glob(self.config.openstack['releases']):
            with io.open(release_yaml, 'r', encoding='utf-8') as f:
                releases.extend(utils.load_fixture(f))

        # inject orchestrator_data into releases if empty
        #
        # NOTE(ikalnitsky): we can drop this block of code when
        #   we got two things done:
        #     * remove `fuelweb` word from default repos
        #     * add this template to `openstack.yaml`
        #     * fill orchestrator_data in nailgun during syncdb
        for release in releases:
            repo_path = \
                'http://{{MASTER_IP}}:8080/{{OPENSTACK_VERSION}}/{OS}/x86_64'\
                .format(OS=release['operating_system'].lower())

            if release['operating_system'].lower() == 'ubuntu':
                repo_path += ' precise main'

            release['orchestrator_data'] = {
                'puppet_manifests_source':
                'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/',

                'puppet_modules_source':
                'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/',

                'repo_metadata': {
                    release['version']: repo_path}}

        return releases

    def _add_base_repos_to_releases(self, releases, existing_releases):
        """Update given releases with orchestrator data of base release.

        :param releases: a list of releases to process
        :param existing_releases: a list of existings releases
        """
        metadata_path = self.config.openstack['metadata']

        # do nothing in case of metadata.yaml absence - just assume
        # that we have full repos
        if not os.path.exists(metadata_path):
            return

        with io.open(metadata_path, 'r', encoding='utf-8') as f:
            metadata = yaml.load(f) or {}

        # keep diff-based releases
        releases = filter(
            lambda r: r['version'] in metadata.get('diff_releases', {}),
            releases)

        # inject repos from base releases
        for release in releases:
            version = release['version']
            base_release = utils.get_base_release(
                release, metadata['diff_releases'][version], existing_releases)

            if base_release is None:
                raise errors.BaseReleaseNotFound(
                    'Could not find a base release - "{0}" - of the '
                    'release "{1}".'.format(
                        metadata['diff_releases'][version], version))

            release['orchestrator_data']['repo_metadata'].update(
                base_release['orchestrator_data']['repo_metadata'])

    @property
    def required_free_space(self):
        spaces = {
            self.config.openstack['puppets']['dst']:
            glob.glob(self.config.openstack['puppets']['src']),

            self.config.openstack['repos']['dst']:
            glob.glob(self.config.openstack['repos']['src'])}

        for dst, srcs in six.iteritems(spaces):
            size = 0
            for src in srcs:
                size += utils.dir_size(src)
            spaces[dst] = size

        return spaces
Ejemplo n.º 5
0
class TestNailgunClient(base.BaseTestCase):
    def setUp(self):
        mock_keystone = mock.MagicMock()
        self.mock_request = mock_keystone.request
        with mock.patch("fuel_upgrade.clients.nailgun_client.KeystoneClient", return_value=mock_keystone):
            self.nailgun = NailgunClient("127.0.0.1", 8000)

    def test_create_release(self):
        # test normal bahavior
        self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }')

        response = self.nailgun.create_release({"name": "Havana on Ubuntu 12.04"})

        self.assertEqual(response, {"id": "42"})

        # test failed result
        self.mock_request.post.return_value.status_code = 409
        self.assertRaises(
            requests.exceptions.HTTPError, self.nailgun.create_release, {"name": "Havana on Ubuntu 12.04"}
        )

    def test_delete_release(self):
        # test normal bahavior
        for status in (200, 204):
            self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content")
            response = self.nailgun.remove_release(42)
            self.assertEqual(response, "No Content")

        # test failed result
        self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict")

        self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_release, 42)

    def test_create_notification(self):
        # test normal bahavior
        self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }')

        response = self.nailgun.create_notification({"topic": "release", "message": "New release available!"})

        self.assertEqual(response, {"id": "42"})

        # test failed result
        self.mock_request.post.return_value.status_code = 409
        self.assertRaises(
            requests.exceptions.HTTPError,
            self.nailgun.create_notification,
            {"topic": "release", "message": "New release available!"},
        )

    def test_delete_notification(self):
        # test normal bahavior
        for status in (200, 204):
            self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content")
            response = self.nailgun.remove_notification(42)
            self.assertEqual(response, "No Content")

        # test failed result
        self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict")

        self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_notification, 42)

    def test_get_tasks(self):
        # test positive cases
        self.mock_request.get.return_value = self.mock_requests_response(200, "[1,2,3]")
        response = self.nailgun.get_tasks()
        self.assertEqual(response, [1, 2, 3])

        # test negative cases
        self.mock_request.get.return_value = self.mock_requests_response(502, "Bad gateway")

        self.assertRaises(requests.exceptions.HTTPError, self.nailgun.get_tasks)

    def test_put_deployment_tasks(self):
        release = {"id": "1"}
        tasks = []
        self.mock_request.put.return_value = self.mock_requests_response(200, "[]")
        response = self.nailgun.put_deployment_tasks(release, tasks)
        self.assertEqual(response, tasks)

        self.mock_request.put.return_value = self.mock_requests_response(502, "Bad gateway")

        self.assertRaises(requests.exceptions.HTTPError, self.nailgun.put_deployment_tasks, release, tasks)
Ejemplo n.º 6
0
class OpenStackUpgrader(UpgradeEngine):
    """OpenStack Upgrader.

    The class is designed to do the following tasks:

    * install manifests in the system
    * add new releases to nailgun's database
    * add notification about new releases
    """

    def __init__(self, *args, **kwargs):
        super(OpenStackUpgrader, self).__init__(*args, **kwargs)

        #: a list of releases to install
        self.releases = self._read_releases()
        #: a nailgun object - api wrapper
        self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])

        self._reset_state()

    def upgrade(self):
        self._reset_state()

        self.install_puppets()
        self.install_releases()
        self.install_versions()

    def rollback(self):
        self.remove_releases()
        self.remove_puppets()
        self.remove_versions()

    def install_puppets(self):
        logger.info('Installing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.copy(source, destination)

    def remove_puppets(self):
        logger.info('Removing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(
                self.config.openstack['puppets']['dst'],
                os.path.basename(source))
            utils.remove(destination)

    def install_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        utils.create_dir_if_not_exists(release_versions_cfg['dst'])
        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.copy(version_file, dst)

    def remove_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        for version_file in versions:
            dst = os.path.join(
                release_versions_cfg['dst'],
                os.path.basename(version_file))
            utils.remove(dst)

    def install_releases(self):
        # add only new releases to nailgun and inject paths to
        # base repo if needed
        existing_releases = self.nailgun.get_releases()
        releases = self._get_unique_releases(self.releases, existing_releases)

        # upload unexisting releases
        for release in releases:
            # register new release
            logger.debug('Register a new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_release(release)
            # save release id for futher possible rollback
            self._rollback_ids['release'].append(response['id'])
            self.upload_release_deployment_tasks(response)
            # add notification abot successfull releases
            logger.debug('Add notification about new release: %s (%s)',
                         release['name'],
                         release['version'])
            response = self.nailgun.create_notification({
                'topic': 'release',
                'message': 'New release available: {0} ({1})'.format(
                    release['name'],
                    release['version'],
                ),
            })
            # save notification id for futher possible rollback
            self._rollback_ids['notification'].append(response['id'])

    def upload_release_deployment_tasks(self, release):
        """Performs os.walk by puppet src, matches all files with tasks
        of given pattern and uploads this for release.

        :param release: dict representation of release
        """
        tasks = []
        release_puppet_path = os.path.join(
            self.config.openstack['puppets']['dst'], release['version'])

        for file_path in utils.iterfiles_filter(
                release_puppet_path,
                self.config.deployment_tasks_file_pattern):

            tasks.extend(utils.read_from_yaml(file_path))

        self.nailgun.put_deployment_tasks(release, tasks)

    def remove_releases(self):
        """Remove all releases that are created by current session.
        """
        for release_id in reversed(self._rollback_ids['release']):
            try:
                logger.debug('Removing release with ID=%s', release_id)
                self.nailgun.remove_release(release_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception('%s', six.text_type(exc))

        for notif_id in reversed(self._rollback_ids['notification']):
            try:
                logger.debug('Removing notification with ID=%s', notif_id)
                self.nailgun.remove_notification(notif_id)
            except (
                requests.exceptions.HTTPError
            ) as exc:
                logger.exception('%s', six.text_type(exc))

    def _reset_state(self):
        """Remove rollback IDs from the arrays.
        """
        #: a list of ids that have to be removed in case of rollback
        self._rollback_ids = {
            'release': [],
            'notification': [],
        }

    @classmethod
    def _get_unique_releases(cls, releases, existing_releases):
        """Returns a list of releases that aren't exist yet.

        :param releases: a list of releases to filter
        :param existing_releases: a list of existing releases
        :returns: a list of unique releases
        """
        existing_releases = [
            (r['name'], r['version']) for r in existing_releases
        ]

        unique = lambda r: (r['name'], r['version']) not in existing_releases
        return [r for r in releases if unique(r)]

    def _read_releases(self):
        """Returns a list of releases in a dict representation.
        """
        releases = []

        # read releases from a set of files
        for release_yaml in glob.glob(self.config.openstack['releases']):
            with io.open(release_yaml, 'r', encoding='utf-8') as f:
                releases.extend(utils.load_fixture(f))

        return releases

    @property
    def required_free_space(self):
        spaces = {
            self.config.openstack['puppets']['dst']:
            glob.glob(self.config.openstack['puppets']['src']), }

        for dst, srcs in six.iteritems(spaces):
            size = 0
            for src in srcs:
                size += utils.dir_size(src)
            spaces[dst] = size

        return spaces
Ejemplo n.º 7
0
class TestNailgunClient(base.BaseTestCase):

    def setUp(self):
        mock_keystone = mock.MagicMock()
        self.mock_request = mock_keystone.request
        with mock.patch(
                'fuel_upgrade.clients.nailgun_client.KeystoneClient',
                return_value=mock_keystone):
            self.nailgun = NailgunClient('127.0.0.1', 8000)

    def test_create_release(self):
        # test normal bahavior
        self.mock_request.post.return_value = self.mock_requests_response(
            201, '{ "id": "42" }')

        response = self.nailgun.create_release({
            'name': 'Havana on Ubuntu 12.04'})

        self.assertEqual(response, {'id': '42'})

        # test failed result
        self.mock_request.post.return_value.status_code = 409
        self.assertRaises(
            requests.exceptions.HTTPError,
            self.nailgun.create_release,
            {'name': 'Havana on Ubuntu 12.04'})

    def test_delete_release(self):
        # test normal bahavior
        for status in (200, 204):
            self.mock_request.delete.return_value = \
                self.mock_requests_response(status, 'No Content')
            response = self.nailgun.remove_release(42)
            self.assertEqual(response, 'No Content')

        # test failed result
        self.mock_request.delete.return_value = self.mock_requests_response(
            409, 'Conflict')

        self.assertRaises(
            requests.exceptions.HTTPError,
            self.nailgun.remove_release,
            42)

    def test_create_notification(self):
        # test normal bahavior
        self.mock_request.post.return_value = self.mock_requests_response(
            201,
            '{ "id": "42" }')

        response = self.nailgun.create_notification({
            'topic': 'release',
            'message': 'New release available!'})

        self.assertEqual(response, {'id': '42'})

        # test failed result
        self.mock_request.post.return_value.status_code = 409
        self.assertRaises(
            requests.exceptions.HTTPError,
            self.nailgun.create_notification,
            {'topic': 'release',
             'message': 'New release available!'})

    def test_delete_notification(self):
        # test normal bahavior
        for status in (200, 204):
            self.mock_request.delete.return_value = \
                self.mock_requests_response(status, 'No Content')
            response = self.nailgun.remove_notification(42)
            self.assertEqual(response, 'No Content')

        # test failed result
        self.mock_request.delete.return_value = self.mock_requests_response(
            409, 'Conflict')

        self.assertRaises(
            requests.exceptions.HTTPError,
            self.nailgun.remove_notification,
            42)

    def test_get_tasks(self):
        # test positive cases
        self.mock_request.get.return_value = self.mock_requests_response(
            200, '[1,2,3]')
        response = self.nailgun.get_tasks()
        self.assertEqual(response, [1, 2, 3])

        # test negative cases
        self.mock_request.get.return_value = self.mock_requests_response(
            502, 'Bad gateway')

        self.assertRaises(
            requests.exceptions.HTTPError, self.nailgun.get_tasks)
Ejemplo n.º 8
0
class OpenStackUpgrader(UpgradeEngine):
    """OpenStack Upgrader.

    The class is designed to do the following tasks:

    * install repos in the system
    * install manifests in the system
    * add new releases to nailgun's database
    * add notification about new releases
    """
    def __init__(self, *args, **kwargs):
        super(OpenStackUpgrader, self).__init__(*args, **kwargs)

        #: a list of releases to install
        self.releases = self._read_releases()
        #: a nailgun object - api wrapper
        self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])

        self._reset_state()

    def upgrade(self):
        self._reset_state()

        self.install_puppets()
        self.install_repos()
        self.install_releases()
        self.install_versions()

    def rollback(self):
        self.remove_releases()
        self.remove_repos()
        self.remove_puppets()
        self.remove_versions()

    def install_puppets(self):
        logger.info('Installing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['puppets']['dst'],
                                       os.path.basename(source))
            utils.copy(source, destination)

    def remove_puppets(self):
        logger.info('Removing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['puppets']['dst'],
                                       os.path.basename(source))
            utils.remove(destination)

    def install_repos(self):
        logger.info('Installing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['repos']['dst'],
                                       os.path.basename(source))
            utils.copy(source, destination)

    def remove_repos(self):
        logger.info('Removing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['repos']['dst'],
                                       os.path.basename(source))
            utils.remove(destination)

    def on_success(self):
        """Do nothing for this engine
        """

    def install_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        utils.create_dir_if_not_exists(release_versions_cfg['dst'])
        for version_file in versions:
            dst = os.path.join(release_versions_cfg['dst'],
                               os.path.basename(version_file))
            utils.copy(version_file, dst)

    def remove_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        for version_file in versions:
            dst = os.path.join(release_versions_cfg['dst'],
                               os.path.basename(version_file))
            utils.remove(dst)

    def install_releases(self):
        # check releases for existing in nailgun side
        releases = self._get_unique_releases(self.releases,
                                             self.nailgun.get_releases())

        # upload unexisting releases
        for release in releases:
            # register new release
            logger.debug('Register a new release: %s (%s)', release['name'],
                         release['version'])
            response = self.nailgun.create_release(release)
            # save release id for futher possible rollback
            self._rollback_ids['release'].append(response['id'])

            # add notification abot successfull releases
            logger.debug('Add notification about new release: %s (%s)',
                         release['name'], release['version'])
            response = self.nailgun.create_notification({
                'topic':
                'release',
                'message':
                'New release available: {0} ({1})'.format(
                    release['name'],
                    release['version'],
                ),
            })
            # save notification id for futher possible rollback
            self._rollback_ids['notification'].append(response['id'])

    def remove_releases(self):
        """Remove all releases that are created by current session.
        """
        for release_id in reversed(self._rollback_ids['release']):
            try:
                logger.debug('Removing release with ID=%s', release_id)
                self.nailgun.remove_release(release_id)
            except (requests.exceptions.HTTPError) as exc:
                logger.exception(six.text_type(exc))

        for notif_id in reversed(self._rollback_ids['notification']):
            try:
                logger.debug('Removing notification with ID=%s', notif_id)
                self.nailgun.remove_notification(notif_id)
            except (requests.exceptions.HTTPError) as exc:
                logger.exception(six.text_type(exc))

    def _reset_state(self):
        """Remove rollback IDs from the arrays.
        """
        #: a list of ids that have to be removed in case of rollback
        self._rollback_ids = {
            'release': [],
            'notification': [],
        }

    @classmethod
    def _get_unique_releases(cls, releases, existing_releases):
        """Returns a list of releases that aren't exist yet.

        :param releases: a list of releases to filter
        :param existing_releases: a list of existing releases
        :returns: a list of unique releases
        """
        existing_releases = [(r['name'], r['version'])
                             for r in existing_releases]

        unique = lambda r: (r['name'], r['version']) not in existing_releases
        return [r for r in releases if unique(r)]

    def _read_releases(self):
        """Returns a list of releases in a dict representation.
        """
        releases = []

        # read releases from a set of files
        for release_yaml in glob.glob(self.config.openstack['releases']):
            with io.open(release_yaml, 'r', encoding='utf-8') as f:
                releases.extend(utils.load_fixture(f))

        # inject orchestrator_data into releases if empty
        for release in releases:
            meta = {
                'version': release['version'],
                'master_ip': self.config.astute['ADMIN_NETWORK']['ipaddress']
            }

            if 'ubuntu' == release['operating_system'].lower():
                repo = 'http://{master_ip}:8080/{version}/ubuntu/x86_64 ' \
                       'precise main'
            else:
                repo = 'http://{master_ip}:8080/{version}/centos/x86_64'

            release['orchestrator_data'] = {
                'puppet_manifests_source':
                ('rsync://{master_ip}:/puppet/{version}/manifests/'.format(
                    **meta)),
                'puppet_modules_source':
                ('rsync://{master_ip}:/puppet/{version}/modules/'.format(
                    **meta)),
                'repo_metadata': {
                    'nailgun': repo.format(**meta)
                }
            }

        return releases

    @property
    def required_free_space(self):
        spaces = {
            self.config.openstack['puppets']['dst']:
            glob.glob(self.config.openstack['puppets']['src']),
            self.config.openstack['repos']['dst']:
            glob.glob(self.config.openstack['repos']['src'])
        }

        for dst, srcs in six.iteritems(spaces):
            size = 0
            for src in srcs:
                size += utils.dir_size(src)
            spaces[dst] = size

        return spaces
Ejemplo n.º 9
0
class OpenStackUpgrader(UpgradeEngine):
    """OpenStack Upgrader.

    The class is designed to do the following tasks:

    * install repos in the system
    * install manifests in the system
    * add new releases to nailgun's database
    * add notification about new releases
    """
    def __init__(self, *args, **kwargs):
        super(OpenStackUpgrader, self).__init__(*args, **kwargs)

        #: a list of releases to install
        self.releases = self._read_releases()
        #: a nailgun object - api wrapper
        self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])

        self._reset_state()

    def upgrade(self):
        self._reset_state()

        self.install_puppets()
        self.install_repos()
        self.install_releases()
        self.install_versions()

    def rollback(self):
        self.remove_releases()
        self.remove_repos()
        self.remove_puppets()
        self.remove_versions()

    def install_puppets(self):
        logger.info('Installing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['puppets']['dst'],
                                       os.path.basename(source))
            utils.copy(source, destination)

    def remove_puppets(self):
        logger.info('Removing puppet manifests...')

        sources = glob.glob(self.config.openstack['puppets']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['puppets']['dst'],
                                       os.path.basename(source))
            utils.remove(destination)

    def install_repos(self):
        logger.info('Installing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['repos']['dst'],
                                       os.path.basename(source))
            utils.copy(source, destination)

    def remove_repos(self):
        logger.info('Removing repositories...')

        sources = glob.glob(self.config.openstack['repos']['src'])
        for source in sources:
            destination = os.path.join(self.config.openstack['repos']['dst'],
                                       os.path.basename(source))
            utils.remove(destination)

    def on_success(self):
        """Do nothing for this engine
        """

    def install_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        utils.create_dir_if_not_exists(release_versions_cfg['dst'])
        for version_file in versions:
            dst = os.path.join(release_versions_cfg['dst'],
                               os.path.basename(version_file))
            utils.copy(version_file, dst)

    def remove_versions(self):
        """Copy openstack release versions
        """
        logger.info('Copy openstack release versions...')
        release_versions_cfg = self.config.openstack['release_versions']
        versions = glob.glob(release_versions_cfg['src'])

        for version_file in versions:
            dst = os.path.join(release_versions_cfg['dst'],
                               os.path.basename(version_file))
            utils.remove(dst)

    def install_releases(self):
        # add only new releases to nailgun and inject paths to
        # base repo if needed
        existing_releases = self.nailgun.get_releases()
        releases = self._get_unique_releases(self.releases, existing_releases)
        self._add_base_repos_to_releases(releases, existing_releases)

        # upload unexisting releases
        for release in releases:
            # register new release
            logger.debug('Register a new release: %s (%s)', release['name'],
                         release['version'])
            response = self.nailgun.create_release(release)
            # save release id for futher possible rollback
            self._rollback_ids['release'].append(response['id'])

            # add notification abot successfull releases
            logger.debug('Add notification about new release: %s (%s)',
                         release['name'], release['version'])
            response = self.nailgun.create_notification({
                'topic':
                'release',
                'message':
                'New release available: {0} ({1})'.format(
                    release['name'],
                    release['version'],
                ),
            })
            # save notification id for futher possible rollback
            self._rollback_ids['notification'].append(response['id'])

    def remove_releases(self):
        """Remove all releases that are created by current session.
        """
        for release_id in reversed(self._rollback_ids['release']):
            try:
                logger.debug('Removing release with ID=%s', release_id)
                self.nailgun.remove_release(release_id)
            except (requests.exceptions.HTTPError) as exc:
                logger.exception('%s', six.text_type(exc))

        for notif_id in reversed(self._rollback_ids['notification']):
            try:
                logger.debug('Removing notification with ID=%s', notif_id)
                self.nailgun.remove_notification(notif_id)
            except (requests.exceptions.HTTPError) as exc:
                logger.exception('%s', six.text_type(exc))

    def _reset_state(self):
        """Remove rollback IDs from the arrays.
        """
        #: a list of ids that have to be removed in case of rollback
        self._rollback_ids = {
            'release': [],
            'notification': [],
        }

    @classmethod
    def _get_unique_releases(cls, releases, existing_releases):
        """Returns a list of releases that aren't exist yet.

        :param releases: a list of releases to filter
        :param existing_releases: a list of existing releases
        :returns: a list of unique releases
        """
        existing_releases = [(r['name'], r['version'])
                             for r in existing_releases]

        unique = lambda r: (r['name'], r['version']) not in existing_releases
        return [r for r in releases if unique(r)]

    def _read_releases(self):
        """Returns a list of releases in a dict representation.
        """
        releases = []

        # read releases from a set of files
        for release_yaml in glob.glob(self.config.openstack['releases']):
            with io.open(release_yaml, 'r', encoding='utf-8') as f:
                releases.extend(utils.load_fixture(f))

        # inject orchestrator_data into releases if empty
        #
        # NOTE(ikalnitsky): we can drop this block of code when
        #   we got two things done:
        #     * remove `fuelweb` word from default repos
        #     * add this template to `openstack.yaml`
        #     * fill orchestrator_data in nailgun during syncdb
        for release in releases:
            repo_path = \
                'http://{{MASTER_IP}}:8080/{{OPENSTACK_VERSION}}/{OS}/x86_64'\
                .format(OS=release['operating_system'].lower())

            if release['operating_system'].lower() == 'ubuntu':
                repo_path += ' precise main'

            release['orchestrator_data'] = {
                'puppet_manifests_source':
                'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/',
                'puppet_modules_source':
                'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/',
                'repo_metadata': {
                    release['version']: repo_path
                }
            }

        return releases

    def _add_base_repos_to_releases(self, releases, existing_releases):
        """Update given releases with orchestrator data of base release.

        :param releases: a list of releases to process
        :param existing_releases: a list of existings releases
        """
        metadata_path = self.config.openstack['metadata']

        # do nothing in case of metadata.yaml absence - just assume
        # that we have full repos
        if not os.path.exists(metadata_path):
            return

        with io.open(metadata_path, 'r', encoding='utf-8') as f:
            metadata = yaml.load(f) or {}

        # keep diff-based releases
        releases = filter(
            lambda r: r['version'] in metadata.get('diff_releases', {}),
            releases)

        # inject repos from base releases
        for release in releases:
            version = release['version']
            base_release = utils.get_base_release(
                release, metadata['diff_releases'][version], existing_releases)

            if base_release is None:
                raise errors.BaseReleaseNotFound(
                    'Could not find a base release - "{0}" - of the '
                    'release "{1}".'.format(metadata['diff_releases'][version],
                                            version))

            release['orchestrator_data']['repo_metadata'].update(
                base_release['orchestrator_data']['repo_metadata'])

    @property
    def required_free_space(self):
        spaces = {
            self.config.openstack['puppets']['dst']:
            glob.glob(self.config.openstack['puppets']['src']),
            self.config.openstack['repos']['dst']:
            glob.glob(self.config.openstack['repos']['src'])
        }

        for dst, srcs in six.iteritems(spaces):
            size = 0
            for src in srcs:
                size += utils.dir_size(src)
            spaces[dst] = size

        return spaces