def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state()
def setUp(self): mock_keystone = mock.MagicMock() self.mock_request = mock_keystone.request with mock.patch( 'fuel_upgrade.clients.nailgun_client.KeystoneClient', return_value=mock_keystone): self.nailgun = NailgunClient('127.0.0.1', 8000)
class CheckNoRunningTasks(BaseBeforeUpgradeChecker): """Checks that there is no running tasks :param config: config object where property endpoints returns dict with nailgun host and port """ def __init__(self, context): nailgun = context.config.endpoints['nginx_nailgun'] self.nailgun_client = NailgunClient(**nailgun) def check(self): """Sends request to nailgun to make sure that there are no running tasks """ logger.info('Check nailgun tasks') tasks = self.nailgun_client.get_tasks() logger.debug('Nailgun tasks {0}'.format(tasks)) running_tasks = filter(lambda t: t['status'] == 'running', tasks) if running_tasks: tasks_msg = [ 'id={0} cluster={1} name={2}'.format(t.get('id'), t.get('cluster'), t.get('name')) for t in running_tasks ] error_msg = 'Cannot run upgrade, tasks are running: {0}'.format( ' '.join(tasks_msg)) raise errors.CannotRunUpgrade(error_msg)
class CheckNoRunningTasks(BaseBeforeUpgradeChecker): """Checks that there is no running tasks :param config: config object where property endpoints returns dict with nailgun host and port """ def __init__(self, context): nailgun = context.config.endpoints['nginx_nailgun'] self.nailgun_client = NailgunClient(**nailgun) def check(self): """Sends request to nailgun to make sure that there are no running tasks """ logger.info('Check nailgun tasks') tasks = self.nailgun_client.get_tasks() logger.debug('Nailgun tasks {0}'.format(tasks)) running_tasks = filter( lambda t: t['status'] == 'running', tasks) if running_tasks: tasks_msg = ['id={0} cluster={1} name={2}'.format( t.get('id'), t.get('cluster'), t.get('name')) for t in running_tasks] error_msg = 'Cannot run upgrade, tasks are running: {0}'.format( ' '.join(tasks_msg)) raise errors.CannotRunUpgrade(error_msg)
def check(self): nailgun_client = NailgunClient(**self.endpoints['nginx_nailgun']) def get_releases(): releases = nailgun_client.get_releases() return releases releases = self.make_safe_request(get_releases) return isinstance(releases, list) and len(releases) > 1
class CheckNoRunningTasks(BaseBeforeUpgradeChecker): """Checks that there is no running tasks :param config: config object where property endpoints returns dict with nailgun host and port """ def __init__(self, context): nailgun = context.config.endpoints["nginx_nailgun"] self.nailgun_client = NailgunClient(**nailgun) def check(self): """Sends request to nailgun to make sure that there are no running tasks """ logger.info("Check nailgun tasks") try: tasks = self.nailgun_client.get_tasks() except requests.ConnectionError: raise errors.NailgunIsNotRunningError("Cannot connect to rest api service") logger.debug("Nailgun tasks %s", tasks) running_tasks = filter(lambda t: t["status"] == "running", tasks) if running_tasks: tasks_msg = [ "id={0} cluster={1} name={2}".format(t.get("id"), t.get("cluster"), t.get("name")) for t in running_tasks ] error_msg = "Cannot run upgrade, tasks are running: {0}".format(" ".join(tasks_msg)) raise errors.CannotRunUpgrade(error_msg)
def __init__(self, context): nailgun = context.config.endpoints['nginx_nailgun'] self.nailgun_client = NailgunClient(**nailgun)
class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # add only new releases to nailgun and inject paths to # base repo if needed existing_releases = self.nailgun.get_releases() releases = self._get_unique_releases(self.releases, existing_releases) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) self.upload_release_deployment_tasks(response) if not release.get('is_deployable', True): continue # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def upload_release_deployment_tasks(self, release): """Performs os.walk by puppet src, matches all files with tasks of given pattern and uploads this for release. :param release: dict representation of release """ tasks = [] release_puppet_path = os.path.join( self.config.openstack['puppets']['dst'], release['version']) for file_path in utils.iterfiles_filter( release_puppet_path, self.config.deployment_tasks_file_pattern): tasks.extend(utils.read_from_yaml(file_path)) self.nailgun.put_deployment_tasks(release, tasks) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [ (r['name'], r['version']) for r in existing_releases ] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) return releases @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), } for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces
class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install repos in the system * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_repos() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_repos() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_repos(self): logger.info('Installing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join( self.config.openstack['repos']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_repos(self): logger.info('Removing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join( self.config.openstack['repos']['dst'], os.path.basename(source)) utils.remove(destination) def on_success(self): """Do nothing for this engine """ def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # check releases for existing in nailgun side releases = self._get_unique_releases( self.releases, self.nailgun.get_releases()) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception(six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception(six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [ (r['name'], r['version']) for r in existing_releases ] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) # inject orchestrator_data into releases if empty for release in releases: meta = { 'version': release['version'], 'master_ip': self.config.astute['ADMIN_NETWORK']['ipaddress']} if 'ubuntu' == release['operating_system'].lower(): repo = 'http://{master_ip}:8080/{version}/ubuntu/x86_64 ' \ 'precise main' else: repo = 'http://{master_ip}:8080/{version}/centos/x86_64' release['orchestrator_data'] = { 'puppet_manifests_source': ( 'rsync://{master_ip}:/puppet/{version}/manifests/'.format( **meta)), 'puppet_modules_source': ( 'rsync://{master_ip}:/puppet/{version}/modules/'.format( **meta)), 'repo_metadata': { 'nailgun': repo.format(**meta)}} return releases @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), self.config.openstack['repos']['dst']: glob.glob(self.config.openstack['repos']['src'])} for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces
class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install repos in the system * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_repos() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_repos() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_repos(self): logger.info('Installing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join( self.config.openstack['repos']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_repos(self): logger.info('Removing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join( self.config.openstack['repos']['dst'], os.path.basename(source)) utils.remove(destination) def on_success(self): """Do nothing for this engine """ def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # add only new releases to nailgun and inject paths to # base repo if needed existing_releases = self.nailgun.get_releases() releases = self._get_unique_releases(self.releases, existing_releases) self._add_base_repos_to_releases(releases, existing_releases) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [ (r['name'], r['version']) for r in existing_releases ] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) # inject orchestrator_data into releases if empty # # NOTE(ikalnitsky): we can drop this block of code when # we got two things done: # * remove `fuelweb` word from default repos # * add this template to `openstack.yaml` # * fill orchestrator_data in nailgun during syncdb for release in releases: repo_path = \ 'http://{{MASTER_IP}}:8080/{{OPENSTACK_VERSION}}/{OS}/x86_64'\ .format(OS=release['operating_system'].lower()) if release['operating_system'].lower() == 'ubuntu': repo_path += ' precise main' release['orchestrator_data'] = { 'puppet_manifests_source': 'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/', 'puppet_modules_source': 'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/', 'repo_metadata': { release['version']: repo_path}} return releases def _add_base_repos_to_releases(self, releases, existing_releases): """Update given releases with orchestrator data of base release. :param releases: a list of releases to process :param existing_releases: a list of existings releases """ metadata_path = self.config.openstack['metadata'] # do nothing in case of metadata.yaml absence - just assume # that we have full repos if not os.path.exists(metadata_path): return with io.open(metadata_path, 'r', encoding='utf-8') as f: metadata = yaml.load(f) or {} # keep diff-based releases releases = filter( lambda r: r['version'] in metadata.get('diff_releases', {}), releases) # inject repos from base releases for release in releases: version = release['version'] base_release = utils.get_base_release( release, metadata['diff_releases'][version], existing_releases) if base_release is None: raise errors.BaseReleaseNotFound( 'Could not find a base release - "{0}" - of the ' 'release "{1}".'.format( metadata['diff_releases'][version], version)) release['orchestrator_data']['repo_metadata'].update( base_release['orchestrator_data']['repo_metadata']) @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), self.config.openstack['repos']['dst']: glob.glob(self.config.openstack['repos']['src'])} for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces
class TestNailgunClient(base.BaseTestCase): def setUp(self): mock_keystone = mock.MagicMock() self.mock_request = mock_keystone.request with mock.patch("fuel_upgrade.clients.nailgun_client.KeystoneClient", return_value=mock_keystone): self.nailgun = NailgunClient("127.0.0.1", 8000) def test_create_release(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }') response = self.nailgun.create_release({"name": "Havana on Ubuntu 12.04"}) self.assertEqual(response, {"id": "42"}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises( requests.exceptions.HTTPError, self.nailgun.create_release, {"name": "Havana on Ubuntu 12.04"} ) def test_delete_release(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content") response = self.nailgun.remove_release(42) self.assertEqual(response, "No Content") # test failed result self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_release, 42) def test_create_notification(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }') response = self.nailgun.create_notification({"topic": "release", "message": "New release available!"}) self.assertEqual(response, {"id": "42"}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises( requests.exceptions.HTTPError, self.nailgun.create_notification, {"topic": "release", "message": "New release available!"}, ) def test_delete_notification(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content") response = self.nailgun.remove_notification(42) self.assertEqual(response, "No Content") # test failed result self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_notification, 42) def test_get_tasks(self): # test positive cases self.mock_request.get.return_value = self.mock_requests_response(200, "[1,2,3]") response = self.nailgun.get_tasks() self.assertEqual(response, [1, 2, 3]) # test negative cases self.mock_request.get.return_value = self.mock_requests_response(502, "Bad gateway") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.get_tasks) def test_put_deployment_tasks(self): release = {"id": "1"} tasks = [] self.mock_request.put.return_value = self.mock_requests_response(200, "[]") response = self.nailgun.put_deployment_tasks(release, tasks) self.assertEqual(response, tasks) self.mock_request.put.return_value = self.mock_requests_response(502, "Bad gateway") self.assertRaises(requests.exceptions.HTTPError, self.nailgun.put_deployment_tasks, release, tasks)
class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install repos in the system * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_repos() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_repos() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join(self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join(self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_repos(self): logger.info('Installing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join(self.config.openstack['repos']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_repos(self): logger.info('Removing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join(self.config.openstack['repos']['dst'], os.path.basename(source)) utils.remove(destination) def on_success(self): """Do nothing for this engine """ def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join(release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join(release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # check releases for existing in nailgun side releases = self._get_unique_releases(self.releases, self.nailgun.get_releases()) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except (requests.exceptions.HTTPError) as exc: logger.exception(six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except (requests.exceptions.HTTPError) as exc: logger.exception(six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [(r['name'], r['version']) for r in existing_releases] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) # inject orchestrator_data into releases if empty for release in releases: meta = { 'version': release['version'], 'master_ip': self.config.astute['ADMIN_NETWORK']['ipaddress'] } if 'ubuntu' == release['operating_system'].lower(): repo = 'http://{master_ip}:8080/{version}/ubuntu/x86_64 ' \ 'precise main' else: repo = 'http://{master_ip}:8080/{version}/centos/x86_64' release['orchestrator_data'] = { 'puppet_manifests_source': ('rsync://{master_ip}:/puppet/{version}/manifests/'.format( **meta)), 'puppet_modules_source': ('rsync://{master_ip}:/puppet/{version}/modules/'.format( **meta)), 'repo_metadata': { 'nailgun': repo.format(**meta) } } return releases @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), self.config.openstack['repos']['dst']: glob.glob(self.config.openstack['repos']['src']) } for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces
class TestNailgunClient(base.BaseTestCase): def setUp(self): mock_keystone = mock.MagicMock() self.mock_request = mock_keystone.request with mock.patch('fuel_upgrade.clients.nailgun_client.KeystoneClient', return_value=mock_keystone): self.nailgun = NailgunClient('127.0.0.1', 8000) def test_create_release(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response( 201, '{ "id": "42" }') response = self.nailgun.create_release( {'name': 'Havana on Ubuntu 12.04'}) self.assertEqual(response, {'id': '42'}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises(requests.exceptions.HTTPError, self.nailgun.create_release, {'name': 'Havana on Ubuntu 12.04'}) def test_delete_release(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = \ self.mock_requests_response(status, 'No Content') response = self.nailgun.remove_release(42) self.assertEqual(response, 'No Content') # test failed result self.mock_request.delete.return_value = self.mock_requests_response( 409, 'Conflict') self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_release, 42) def test_create_notification(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response( 201, '{ "id": "42" }') response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available!' }) self.assertEqual(response, {'id': '42'}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises(requests.exceptions.HTTPError, self.nailgun.create_notification, { 'topic': 'release', 'message': 'New release available!' }) def test_delete_notification(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = \ self.mock_requests_response(status, 'No Content') response = self.nailgun.remove_notification(42) self.assertEqual(response, 'No Content') # test failed result self.mock_request.delete.return_value = self.mock_requests_response( 409, 'Conflict') self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_notification, 42) def test_get_tasks(self): # test positive cases self.mock_request.get.return_value = self.mock_requests_response( 200, '[1,2,3]') response = self.nailgun.get_tasks() self.assertEqual(response, [1, 2, 3]) # test negative cases self.mock_request.get.return_value = self.mock_requests_response( 502, 'Bad gateway') self.assertRaises(requests.exceptions.HTTPError, self.nailgun.get_tasks)
class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join( self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join( release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # add only new releases to nailgun and inject paths to # base repo if needed existing_releases = self.nailgun.get_releases() releases = self._get_unique_releases(self.releases, existing_releases) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) self.upload_release_deployment_tasks(response) # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def upload_release_deployment_tasks(self, release): """Performs os.walk by puppet src, matches all files with tasks of given pattern and uploads this for release. :param release: dict representation of release """ tasks = [] release_puppet_path = os.path.join( self.config.openstack['puppets']['dst'], release['version']) for file_path in utils.iterfiles_filter( release_puppet_path, self.config.deployment_tasks_file_pattern): tasks.extend(utils.read_from_yaml(file_path)) self.nailgun.put_deployment_tasks(release, tasks) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except ( requests.exceptions.HTTPError ) as exc: logger.exception('%s', six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [ (r['name'], r['version']) for r in existing_releases ] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) return releases @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), } for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces
class TestNailgunClient(base.BaseTestCase): def setUp(self): mock_keystone = mock.MagicMock() self.mock_request = mock_keystone.request with mock.patch( 'fuel_upgrade.clients.nailgun_client.KeystoneClient', return_value=mock_keystone): self.nailgun = NailgunClient('127.0.0.1', 8000) def test_create_release(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response( 201, '{ "id": "42" }') response = self.nailgun.create_release({ 'name': 'Havana on Ubuntu 12.04'}) self.assertEqual(response, {'id': '42'}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises( requests.exceptions.HTTPError, self.nailgun.create_release, {'name': 'Havana on Ubuntu 12.04'}) def test_delete_release(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = \ self.mock_requests_response(status, 'No Content') response = self.nailgun.remove_release(42) self.assertEqual(response, 'No Content') # test failed result self.mock_request.delete.return_value = self.mock_requests_response( 409, 'Conflict') self.assertRaises( requests.exceptions.HTTPError, self.nailgun.remove_release, 42) def test_create_notification(self): # test normal bahavior self.mock_request.post.return_value = self.mock_requests_response( 201, '{ "id": "42" }') response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available!'}) self.assertEqual(response, {'id': '42'}) # test failed result self.mock_request.post.return_value.status_code = 409 self.assertRaises( requests.exceptions.HTTPError, self.nailgun.create_notification, {'topic': 'release', 'message': 'New release available!'}) def test_delete_notification(self): # test normal bahavior for status in (200, 204): self.mock_request.delete.return_value = \ self.mock_requests_response(status, 'No Content') response = self.nailgun.remove_notification(42) self.assertEqual(response, 'No Content') # test failed result self.mock_request.delete.return_value = self.mock_requests_response( 409, 'Conflict') self.assertRaises( requests.exceptions.HTTPError, self.nailgun.remove_notification, 42) def test_get_tasks(self): # test positive cases self.mock_request.get.return_value = self.mock_requests_response( 200, '[1,2,3]') response = self.nailgun.get_tasks() self.assertEqual(response, [1, 2, 3]) # test negative cases self.mock_request.get.return_value = self.mock_requests_response( 502, 'Bad gateway') self.assertRaises( requests.exceptions.HTTPError, self.nailgun.get_tasks)
def setUp(self): mock_keystone = mock.MagicMock() self.mock_request = mock_keystone.request with mock.patch('fuel_upgrade.clients.nailgun_client.KeystoneClient', return_value=mock_keystone): self.nailgun = NailgunClient('127.0.0.1', 8000)
class OpenStackUpgrader(UpgradeEngine): """OpenStack Upgrader. The class is designed to do the following tasks: * install repos in the system * install manifests in the system * add new releases to nailgun's database * add notification about new releases """ def __init__(self, *args, **kwargs): super(OpenStackUpgrader, self).__init__(*args, **kwargs) #: a list of releases to install self.releases = self._read_releases() #: a nailgun object - api wrapper self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun']) self._reset_state() def upgrade(self): self._reset_state() self.install_puppets() self.install_repos() self.install_releases() self.install_versions() def rollback(self): self.remove_releases() self.remove_repos() self.remove_puppets() self.remove_versions() def install_puppets(self): logger.info('Installing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join(self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_puppets(self): logger.info('Removing puppet manifests...') sources = glob.glob(self.config.openstack['puppets']['src']) for source in sources: destination = os.path.join(self.config.openstack['puppets']['dst'], os.path.basename(source)) utils.remove(destination) def install_repos(self): logger.info('Installing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join(self.config.openstack['repos']['dst'], os.path.basename(source)) utils.copy(source, destination) def remove_repos(self): logger.info('Removing repositories...') sources = glob.glob(self.config.openstack['repos']['src']) for source in sources: destination = os.path.join(self.config.openstack['repos']['dst'], os.path.basename(source)) utils.remove(destination) def on_success(self): """Do nothing for this engine """ def install_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) utils.create_dir_if_not_exists(release_versions_cfg['dst']) for version_file in versions: dst = os.path.join(release_versions_cfg['dst'], os.path.basename(version_file)) utils.copy(version_file, dst) def remove_versions(self): """Copy openstack release versions """ logger.info('Copy openstack release versions...') release_versions_cfg = self.config.openstack['release_versions'] versions = glob.glob(release_versions_cfg['src']) for version_file in versions: dst = os.path.join(release_versions_cfg['dst'], os.path.basename(version_file)) utils.remove(dst) def install_releases(self): # add only new releases to nailgun and inject paths to # base repo if needed existing_releases = self.nailgun.get_releases() releases = self._get_unique_releases(self.releases, existing_releases) self._add_base_repos_to_releases(releases, existing_releases) # upload unexisting releases for release in releases: # register new release logger.debug('Register a new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_release(release) # save release id for futher possible rollback self._rollback_ids['release'].append(response['id']) # add notification abot successfull releases logger.debug('Add notification about new release: %s (%s)', release['name'], release['version']) response = self.nailgun.create_notification({ 'topic': 'release', 'message': 'New release available: {0} ({1})'.format( release['name'], release['version'], ), }) # save notification id for futher possible rollback self._rollback_ids['notification'].append(response['id']) def remove_releases(self): """Remove all releases that are created by current session. """ for release_id in reversed(self._rollback_ids['release']): try: logger.debug('Removing release with ID=%s', release_id) self.nailgun.remove_release(release_id) except (requests.exceptions.HTTPError) as exc: logger.exception('%s', six.text_type(exc)) for notif_id in reversed(self._rollback_ids['notification']): try: logger.debug('Removing notification with ID=%s', notif_id) self.nailgun.remove_notification(notif_id) except (requests.exceptions.HTTPError) as exc: logger.exception('%s', six.text_type(exc)) def _reset_state(self): """Remove rollback IDs from the arrays. """ #: a list of ids that have to be removed in case of rollback self._rollback_ids = { 'release': [], 'notification': [], } @classmethod def _get_unique_releases(cls, releases, existing_releases): """Returns a list of releases that aren't exist yet. :param releases: a list of releases to filter :param existing_releases: a list of existing releases :returns: a list of unique releases """ existing_releases = [(r['name'], r['version']) for r in existing_releases] unique = lambda r: (r['name'], r['version']) not in existing_releases return [r for r in releases if unique(r)] def _read_releases(self): """Returns a list of releases in a dict representation. """ releases = [] # read releases from a set of files for release_yaml in glob.glob(self.config.openstack['releases']): with io.open(release_yaml, 'r', encoding='utf-8') as f: releases.extend(utils.load_fixture(f)) # inject orchestrator_data into releases if empty # # NOTE(ikalnitsky): we can drop this block of code when # we got two things done: # * remove `fuelweb` word from default repos # * add this template to `openstack.yaml` # * fill orchestrator_data in nailgun during syncdb for release in releases: repo_path = \ 'http://{{MASTER_IP}}:8080/{{OPENSTACK_VERSION}}/{OS}/x86_64'\ .format(OS=release['operating_system'].lower()) if release['operating_system'].lower() == 'ubuntu': repo_path += ' precise main' release['orchestrator_data'] = { 'puppet_manifests_source': 'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/', 'puppet_modules_source': 'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/', 'repo_metadata': { release['version']: repo_path } } return releases def _add_base_repos_to_releases(self, releases, existing_releases): """Update given releases with orchestrator data of base release. :param releases: a list of releases to process :param existing_releases: a list of existings releases """ metadata_path = self.config.openstack['metadata'] # do nothing in case of metadata.yaml absence - just assume # that we have full repos if not os.path.exists(metadata_path): return with io.open(metadata_path, 'r', encoding='utf-8') as f: metadata = yaml.load(f) or {} # keep diff-based releases releases = filter( lambda r: r['version'] in metadata.get('diff_releases', {}), releases) # inject repos from base releases for release in releases: version = release['version'] base_release = utils.get_base_release( release, metadata['diff_releases'][version], existing_releases) if base_release is None: raise errors.BaseReleaseNotFound( 'Could not find a base release - "{0}" - of the ' 'release "{1}".'.format(metadata['diff_releases'][version], version)) release['orchestrator_data']['repo_metadata'].update( base_release['orchestrator_data']['repo_metadata']) @property def required_free_space(self): spaces = { self.config.openstack['puppets']['dst']: glob.glob(self.config.openstack['puppets']['src']), self.config.openstack['repos']['dst']: glob.glob(self.config.openstack['repos']['src']) } for dst, srcs in six.iteritems(spaces): size = 0 for src in srcs: size += utils.dir_size(src) spaces[dst] = size return spaces