class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # add only new releases to nailgun and inject paths to # base repo if needed existing_releases = self.nailgun.get_releases() releases = self._get_unique_releases(self.releases, existing_releases) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) self.upload_release_deployment_tasks(response) if not release.get('is_deployable', True): continue # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def upload_release_deployment_tasks(self, release): """Performs os.walk by puppet src, matches all files with tasks of given pattern and uploads this for release. :param release: dict representation of release """ tasks = [] release_puppet_path = os.path.join( self.config.openstack['puppets']['dst'], release['version']) for file_path in utils.iterfiles_filter( release_puppet_path, self.config.deployment_tasks_file_pattern): tasks.extend(utils.read_from_yaml(file_path)) self.nailgun.put_deployment_tasks(release, tasks) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [ (r['name'], r['version']) for r in existing_releases ] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) return releases @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), } for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces
class TestNailgunClient(base.BaseTestCase): def setUp(self): mock_keystone = mock.MagicMock() self.mock_request = mock_keystone.request with mock.patch('fuel_upgrade.clients.nailgun_client.KeystoneClient', return_value=mock_keystone): self.nailgun = NailgunClient('127.0.0.1', 8000) def test_create_release(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response( 201, '{ "id": "42" }') response = self.nailgun.create_release( {'name': 'Havana on Ubuntu 12.04'}) self.assertEqual(response, {'id': '42'}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises(requests.exceptions.HTTPError, self.nailgun.create_release, {'name': 'Havana on Ubuntu 12.04'}) def test_delete_release(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = \ self.mock_requests_response(status, 'No Content') response = self.nailgun.remove_release(42) self.assertEqual(response, 'No Content') # test failed result self.mock_request.delete.return_value = self.mock_requests_response( 409, 'Conflict') self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_release, 42) def test_create_notification(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response( 201, '{ "id": "42" }') response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available!' }) self.assertEqual(response, {'id': '42'}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises(requests.exceptions.HTTPError, self.nailgun.create_notification, { 'topic': 'release', 'message': 'New release available!' }) def test_delete_notification(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = \ self.mock_requests_response(status, 'No Content') response = self.nailgun.remove_notification(42) self.assertEqual(response, 'No Content') # test failed result self.mock_request.delete.return_value = self.mock_requests_response( 409, 'Conflict') self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_notification, 42) def test_get_tasks(self): # test positive cases self.mock_request.get.return_value = self.mock_requests_response( 200, '[1,2,3]') response = self.nailgun.get_tasks() self.assertEqual(response, [1, 2, 3]) # test negative cases self.mock_request.get.return_value = self.mock_requests_response( 502, 'Bad gateway') self.assertRaises(requests.exceptions.HTTPError, self.nailgun.get_tasks) def test_put_deployment_tasks(self): release = {'id': '1'} tasks = [] self.mock_request.put.return_value = self.mock_requests_response( 200, '[]') response = self.nailgun.put_deployment_tasks(release, tasks) self.assertEqual(response, tasks) self.mock_request.put.return_value = self.mock_requests_response( 502, 'Bad gateway') self.assertRaises(requests.exceptions.HTTPError, self.nailgun.put_deployment_tasks, release, tasks)
class TestNailgunClient(base.BaseTestCase): def setUp(self): mock_keystone = mock.MagicMock() self.mock_request = mock_keystone.request with mock.patch("fuel_upgrade.clients.nailgun_client.KeystoneClient", return_value=mock_keystone): self.nailgun = NailgunClient("127.0.0.1", 8000) def test_create_release(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }') response = self.nailgun.create_release({"name": "Havana on Ubuntu 12.04"}) self.assertEqual(response, {"id": "42"}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises( requests.exceptions.HTTPError, self.nailgun.create_release, {"name": "Havana on Ubuntu 12.04"} ) def test_delete_release(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content") response = self.nailgun.remove_release(42) self.assertEqual(response, "No Content") # test failed result self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_release, 42) def test_create_notification(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }') response = self.nailgun.create_notification({"topic": "release", "message": "New release available!"}) self.assertEqual(response, {"id": "42"}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises( requests.exceptions.HTTPError, self.nailgun.create_notification, {"topic": "release", "message": "New release available!"}, ) def test_delete_notification(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content") response = self.nailgun.remove_notification(42) self.assertEqual(response, "No Content") # test failed result self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_notification, 42) def test_get_tasks(self): # test positive cases self.mock_request.get.return_value = self.mock_requests_response(200, "[1,2,3]") response = self.nailgun.get_tasks() self.assertEqual(response, [1, 2, 3]) # test negative cases self.mock_request.get.return_value = self.mock_requests_response(502, "Bad gateway") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.get_tasks) def test_put_deployment_tasks(self): release = {"id": "1"} tasks = [] self.mock_request.put.return_value = self.mock_requests_response(200, "[]") response = self.nailgun.put_deployment_tasks(release, tasks) self.assertEqual(response, tasks) self.mock_request.put.return_value = self.mock_requests_response(502, "Bad gateway") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.put_deployment_tasks, release, tasks)
class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # add only new releases to nailgun and inject paths to # base repo if needed existing_releases = self.nailgun.get_releases() releases = self._get_unique_releases(self.releases, existing_releases) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) self.upload_release_deployment_tasks(response) # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def upload_release_deployment_tasks(self, release): """Performs os.walk by puppet src, matches all files with tasks of given pattern and uploads this for release. :param release: dict representation of release """ tasks = [] release_puppet_path = os.path.join( self.config.openstack['puppets']['dst'], release['version']) for file_path in utils.iterfiles_filter( release_puppet_path, self.config.deployment_tasks_file_pattern): tasks.extend(utils.read_from_yaml(file_path)) self.nailgun.put_deployment_tasks(release, tasks) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [ (r['name'], r['version']) for r in existing_releases ] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) return releases @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), } for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces