class OpenStackUpgrader(UpgradeEngine):
"""OpenStack Upgrader.
The class is designed to do the following tasks:
* install manifests in the system
* add new releases to nailgun's database
* add notification about new releases
"""
def __init__(self, *args, **kwargs):
super(OpenStackUpgrader, self).__init__(*args, **kwargs)
#: a list of releases to install
self.releases = self._read_releases()
#: a nailgun object - api wrapper
self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])
self._reset_state()
def upgrade(self):
self._reset_state()
self.install_puppets()
self.install_releases()
self.install_versions()
def rollback(self):
self.remove_releases()
self.remove_puppets()
self.remove_versions()
def install_puppets(self):
logger.info('Installing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_puppets(self):
logger.info('Removing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.remove(destination)
def install_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
utils.create_dir_if_not_exists(release_versions_cfg['dst'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.copy(version_file, dst)
def remove_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.remove(dst)
def install_releases(self):
# add only new releases to nailgun and inject paths to
# base repo if needed
existing_releases = self.nailgun.get_releases()
releases = self._get_unique_releases(self.releases, existing_releases)
# upload unexisting releases
for release in releases:
# register new release
logger.debug('Register a new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_release(release)
# save release id for futher possible rollback
self._rollback_ids['release'].append(response['id'])
self.upload_release_deployment_tasks(response)
if not release.get('is_deployable', True):
continue
# add notification abot successfull releases
logger.debug('Add notification about new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_notification({
'topic': 'release',
'message': 'New release available: {0} ({1})'.format(
release['name'],
release['version'],
),
})
# save notification id for futher possible rollback
self._rollback_ids['notification'].append(response['id'])
def upload_release_deployment_tasks(self, release):
"""Performs os.walk by puppet src, matches all files with tasks
of given pattern and uploads this for release.
:param release: dict representation of release
"""
tasks = []
release_puppet_path = os.path.join(
self.config.openstack['puppets']['dst'], release['version'])
for file_path in utils.iterfiles_filter(
release_puppet_path,
self.config.deployment_tasks_file_pattern):
tasks.extend(utils.read_from_yaml(file_path))
self.nailgun.put_deployment_tasks(release, tasks)
def remove_releases(self):
"""Remove all releases that are created by current session.
"""
for release_id in reversed(self._rollback_ids['release']):
try:
logger.debug('Removing release with ID=%s', release_id)
self.nailgun.remove_release(release_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception('%s', six.text_type(exc))
for notif_id in reversed(self._rollback_ids['notification']):
try:
logger.debug('Removing notification with ID=%s', notif_id)
self.nailgun.remove_notification(notif_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception('%s', six.text_type(exc))
def _reset_state(self):
"""Remove rollback IDs from the arrays.
"""
#: a list of ids that have to be removed in case of rollback
self._rollback_ids = {
'release': [],
'notification': [],
}
@classmethod
def _get_unique_releases(cls, releases, existing_releases):
"""Returns a list of releases that aren't exist yet.
:param releases: a list of releases to filter
:param existing_releases: a list of existing releases
:returns: a list of unique releases
"""
existing_releases = [
(r['name'], r['version']) for r in existing_releases
]
unique = lambda r: (r['name'], r['version']) not in existing_releases
return [r for r in releases if unique(r)]
def _read_releases(self):
"""Returns a list of releases in a dict representation.
"""
releases = []
# read releases from a set of files
for release_yaml in glob.glob(self.config.openstack['releases']):
with io.open(release_yaml, 'r', encoding='utf-8') as f:
releases.extend(utils.load_fixture(f))
return releases
@property
def required_free_space(self):
spaces = {
self.config.openstack['puppets']['dst']:
glob.glob(self.config.openstack['puppets']['src']), }
for dst, srcs in six.iteritems(spaces):
size = 0
for src in srcs:
size += utils.dir_size(src)
spaces[dst] = size
return spaces
class TestNailgunClient(base.BaseTestCase):
def setUp(self):
mock_keystone = mock.MagicMock()
self.mock_request = mock_keystone.request
with mock.patch('fuel_upgrade.clients.nailgun_client.KeystoneClient',
return_value=mock_keystone):
self.nailgun = NailgunClient('127.0.0.1', 8000)
def test_create_release(self):
# test normal bahavior
self.mock_request.post.return_value = self.mock_requests_response(
201, '{ "id": "42" }')
response = self.nailgun.create_release(
{'name': 'Havana on Ubuntu 12.04'})
self.assertEqual(response, {'id': '42'})
# test failed result
self.mock_request.post.return_value.status_code = 409
self.assertRaises(requests.exceptions.HTTPError,
self.nailgun.create_release,
{'name': 'Havana on Ubuntu 12.04'})
def test_delete_release(self):
# test normal bahavior
for status in (200, 204):
self.mock_request.delete.return_value = \
self.mock_requests_response(status, 'No Content')
response = self.nailgun.remove_release(42)
self.assertEqual(response, 'No Content')
# test failed result
self.mock_request.delete.return_value = self.mock_requests_response(
409, 'Conflict')
self.assertRaises(requests.exceptions.HTTPError,
self.nailgun.remove_release, 42)
def test_create_notification(self):
# test normal bahavior
self.mock_request.post.return_value = self.mock_requests_response(
201, '{ "id": "42" }')
response = self.nailgun.create_notification({
'topic':
'release',
'message':
'New release available!'
})
self.assertEqual(response, {'id': '42'})
# test failed result
self.mock_request.post.return_value.status_code = 409
self.assertRaises(requests.exceptions.HTTPError,
self.nailgun.create_notification, {
'topic': 'release',
'message': 'New release available!'
})
def test_delete_notification(self):
# test normal bahavior
for status in (200, 204):
self.mock_request.delete.return_value = \
self.mock_requests_response(status, 'No Content')
response = self.nailgun.remove_notification(42)
self.assertEqual(response, 'No Content')
# test failed result
self.mock_request.delete.return_value = self.mock_requests_response(
409, 'Conflict')
self.assertRaises(requests.exceptions.HTTPError,
self.nailgun.remove_notification, 42)
def test_get_tasks(self):
# test positive cases
self.mock_request.get.return_value = self.mock_requests_response(
200, '[1,2,3]')
response = self.nailgun.get_tasks()
self.assertEqual(response, [1, 2, 3])
# test negative cases
self.mock_request.get.return_value = self.mock_requests_response(
502, 'Bad gateway')
self.assertRaises(requests.exceptions.HTTPError,
self.nailgun.get_tasks)
class OpenStackUpgrader(UpgradeEngine):
"""OpenStack Upgrader.
The class is designed to do the following tasks:
* install repos in the system
* install manifests in the system
* add new releases to nailgun's database
* add notification about new releases
"""
def __init__(self, *args, **kwargs):
super(OpenStackUpgrader, self).__init__(*args, **kwargs)
#: a list of releases to install
self.releases = self._read_releases()
#: a nailgun object - api wrapper
self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])
self._reset_state()
def upgrade(self):
self._reset_state()
self.install_puppets()
self.install_repos()
self.install_releases()
self.install_versions()
def rollback(self):
self.remove_releases()
self.remove_repos()
self.remove_puppets()
self.remove_versions()
def install_puppets(self):
logger.info('Installing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_puppets(self):
logger.info('Removing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.remove(destination)
def install_repos(self):
logger.info('Installing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_repos(self):
logger.info('Removing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.remove(destination)
def on_success(self):
"""Do nothing for this engine
"""
def install_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
utils.create_dir_if_not_exists(release_versions_cfg['dst'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.copy(version_file, dst)
def remove_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.remove(dst)
def install_releases(self):
# check releases for existing in nailgun side
releases = self._get_unique_releases(
self.releases, self.nailgun.get_releases())
# upload unexisting releases
for release in releases:
# register new release
logger.debug('Register a new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_release(release)
# save release id for futher possible rollback
self._rollback_ids['release'].append(response['id'])
# add notification abot successfull releases
logger.debug('Add notification about new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_notification({
'topic': 'release',
'message': 'New release available: {0} ({1})'.format(
release['name'],
release['version'],
),
})
# save notification id for futher possible rollback
self._rollback_ids['notification'].append(response['id'])
def remove_releases(self):
"""Remove all releases that are created by current session.
"""
for release_id in reversed(self._rollback_ids['release']):
try:
logger.debug('Removing release with ID=%s', release_id)
self.nailgun.remove_release(release_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception(six.text_type(exc))
for notif_id in reversed(self._rollback_ids['notification']):
try:
logger.debug('Removing notification with ID=%s', notif_id)
self.nailgun.remove_notification(notif_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception(six.text_type(exc))
def _reset_state(self):
"""Remove rollback IDs from the arrays.
"""
#: a list of ids that have to be removed in case of rollback
self._rollback_ids = {
'release': [],
'notification': [],
}
@classmethod
def _get_unique_releases(cls, releases, existing_releases):
"""Returns a list of releases that aren't exist yet.
:param releases: a list of releases to filter
:param existing_releases: a list of existing releases
:returns: a list of unique releases
"""
existing_releases = [
(r['name'], r['version']) for r in existing_releases
]
unique = lambda r: (r['name'], r['version']) not in existing_releases
return [r for r in releases if unique(r)]
def _read_releases(self):
"""Returns a list of releases in a dict representation.
"""
releases = []
# read releases from a set of files
for release_yaml in glob.glob(self.config.openstack['releases']):
with io.open(release_yaml, 'r', encoding='utf-8') as f:
releases.extend(utils.load_fixture(f))
# inject orchestrator_data into releases if empty
for release in releases:
meta = {
'version': release['version'],
'master_ip': self.config.astute['ADMIN_NETWORK']['ipaddress']}
if 'ubuntu' == release['operating_system'].lower():
repo = 'http://{master_ip}:8080/{version}/ubuntu/x86_64 ' \
'precise main'
else:
repo = 'http://{master_ip}:8080/{version}/centos/x86_64'
release['orchestrator_data'] = {
'puppet_manifests_source': (
'rsync://{master_ip}:/puppet/{version}/manifests/'.format(
**meta)),
'puppet_modules_source': (
'rsync://{master_ip}:/puppet/{version}/modules/'.format(
**meta)),
'repo_metadata': {
'nailgun': repo.format(**meta)}}
return releases
@property
def required_free_space(self):
spaces = {
self.config.openstack['puppets']['dst']:
glob.glob(self.config.openstack['puppets']['src']),
self.config.openstack['repos']['dst']:
glob.glob(self.config.openstack['repos']['src'])}
for dst, srcs in six.iteritems(spaces):
size = 0
for src in srcs:
size += utils.dir_size(src)
spaces[dst] = size
return spaces
class OpenStackUpgrader(UpgradeEngine):
"""OpenStack Upgrader.
The class is designed to do the following tasks:
* install repos in the system
* install manifests in the system
* add new releases to nailgun's database
* add notification about new releases
"""
def __init__(self, *args, **kwargs):
super(OpenStackUpgrader, self).__init__(*args, **kwargs)
#: a list of releases to install
self.releases = self._read_releases()
#: a nailgun object - api wrapper
self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])
self._reset_state()
def upgrade(self):
self._reset_state()
self.install_puppets()
self.install_repos()
self.install_releases()
self.install_versions()
def rollback(self):
self.remove_releases()
self.remove_repos()
self.remove_puppets()
self.remove_versions()
def install_puppets(self):
logger.info('Installing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_puppets(self):
logger.info('Removing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.remove(destination)
def install_repos(self):
logger.info('Installing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_repos(self):
logger.info('Removing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.remove(destination)
def on_success(self):
"""Do nothing for this engine
"""
def install_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
utils.create_dir_if_not_exists(release_versions_cfg['dst'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.copy(version_file, dst)
def remove_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.remove(dst)
def install_releases(self):
# add only new releases to nailgun and inject paths to
# base repo if needed
existing_releases = self.nailgun.get_releases()
releases = self._get_unique_releases(self.releases, existing_releases)
self._add_base_repos_to_releases(releases, existing_releases)
# upload unexisting releases
for release in releases:
# register new release
logger.debug('Register a new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_release(release)
# save release id for futher possible rollback
self._rollback_ids['release'].append(response['id'])
# add notification abot successfull releases
logger.debug('Add notification about new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_notification({
'topic': 'release',
'message': 'New release available: {0} ({1})'.format(
release['name'],
release['version'],
),
})
# save notification id for futher possible rollback
self._rollback_ids['notification'].append(response['id'])
def remove_releases(self):
"""Remove all releases that are created by current session.
"""
for release_id in reversed(self._rollback_ids['release']):
try:
logger.debug('Removing release with ID=%s', release_id)
self.nailgun.remove_release(release_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception('%s', six.text_type(exc))
for notif_id in reversed(self._rollback_ids['notification']):
try:
logger.debug('Removing notification with ID=%s', notif_id)
self.nailgun.remove_notification(notif_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception('%s', six.text_type(exc))
def _reset_state(self):
"""Remove rollback IDs from the arrays.
"""
#: a list of ids that have to be removed in case of rollback
self._rollback_ids = {
'release': [],
'notification': [],
}
@classmethod
def _get_unique_releases(cls, releases, existing_releases):
"""Returns a list of releases that aren't exist yet.
:param releases: a list of releases to filter
:param existing_releases: a list of existing releases
:returns: a list of unique releases
"""
existing_releases = [
(r['name'], r['version']) for r in existing_releases
]
unique = lambda r: (r['name'], r['version']) not in existing_releases
return [r for r in releases if unique(r)]
def _read_releases(self):
"""Returns a list of releases in a dict representation.
"""
releases = []
# read releases from a set of files
for release_yaml in glob.glob(self.config.openstack['releases']):
with io.open(release_yaml, 'r', encoding='utf-8') as f:
releases.extend(utils.load_fixture(f))
# inject orchestrator_data into releases if empty
#
# NOTE(ikalnitsky): we can drop this block of code when
# we got two things done:
# * remove `fuelweb` word from default repos
# * add this template to `openstack.yaml`
# * fill orchestrator_data in nailgun during syncdb
for release in releases:
repo_path = \
'http://{{MASTER_IP}}:8080/{{OPENSTACK_VERSION}}/{OS}/x86_64'\
.format(OS=release['operating_system'].lower())
if release['operating_system'].lower() == 'ubuntu':
repo_path += ' precise main'
release['orchestrator_data'] = {
'puppet_manifests_source':
'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/',
'puppet_modules_source':
'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/',
'repo_metadata': {
release['version']: repo_path}}
return releases
def _add_base_repos_to_releases(self, releases, existing_releases):
"""Update given releases with orchestrator data of base release.
:param releases: a list of releases to process
:param existing_releases: a list of existings releases
"""
metadata_path = self.config.openstack['metadata']
# do nothing in case of metadata.yaml absence - just assume
# that we have full repos
if not os.path.exists(metadata_path):
return
with io.open(metadata_path, 'r', encoding='utf-8') as f:
metadata = yaml.load(f) or {}
# keep diff-based releases
releases = filter(
lambda r: r['version'] in metadata.get('diff_releases', {}),
releases)
# inject repos from base releases
for release in releases:
version = release['version']
base_release = utils.get_base_release(
release, metadata['diff_releases'][version], existing_releases)
if base_release is None:
raise errors.BaseReleaseNotFound(
'Could not find a base release - "{0}" - of the '
'release "{1}".'.format(
metadata['diff_releases'][version], version))
release['orchestrator_data']['repo_metadata'].update(
base_release['orchestrator_data']['repo_metadata'])
@property
def required_free_space(self):
spaces = {
self.config.openstack['puppets']['dst']:
glob.glob(self.config.openstack['puppets']['src']),
self.config.openstack['repos']['dst']:
glob.glob(self.config.openstack['repos']['src'])}
for dst, srcs in six.iteritems(spaces):
size = 0
for src in srcs:
size += utils.dir_size(src)
spaces[dst] = size
return spaces
class TestNailgunClient(base.BaseTestCase):
def setUp(self):
mock_keystone = mock.MagicMock()
self.mock_request = mock_keystone.request
with mock.patch("fuel_upgrade.clients.nailgun_client.KeystoneClient", return_value=mock_keystone):
self.nailgun = NailgunClient("127.0.0.1", 8000)
def test_create_release(self):
# test normal bahavior
self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }')
response = self.nailgun.create_release({"name": "Havana on Ubuntu 12.04"})
self.assertEqual(response, {"id": "42"})
# test failed result
self.mock_request.post.return_value.status_code = 409
self.assertRaises(
requests.exceptions.HTTPError, self.nailgun.create_release, {"name": "Havana on Ubuntu 12.04"}
)
def test_delete_release(self):
# test normal bahavior
for status in (200, 204):
self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content")
response = self.nailgun.remove_release(42)
self.assertEqual(response, "No Content")
# test failed result
self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict")
self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_release, 42)
def test_create_notification(self):
# test normal bahavior
self.mock_request.post.return_value = self.mock_requests_response(201, '{ "id": "42" }')
response = self.nailgun.create_notification({"topic": "release", "message": "New release available!"})
self.assertEqual(response, {"id": "42"})
# test failed result
self.mock_request.post.return_value.status_code = 409
self.assertRaises(
requests.exceptions.HTTPError,
self.nailgun.create_notification,
{"topic": "release", "message": "New release available!"},
)
def test_delete_notification(self):
# test normal bahavior
for status in (200, 204):
self.mock_request.delete.return_value = self.mock_requests_response(status, "No Content")
response = self.nailgun.remove_notification(42)
self.assertEqual(response, "No Content")
# test failed result
self.mock_request.delete.return_value = self.mock_requests_response(409, "Conflict")
self.assertRaises(requests.exceptions.HTTPError, self.nailgun.remove_notification, 42)
def test_get_tasks(self):
# test positive cases
self.mock_request.get.return_value = self.mock_requests_response(200, "[1,2,3]")
response = self.nailgun.get_tasks()
self.assertEqual(response, [1, 2, 3])
# test negative cases
self.mock_request.get.return_value = self.mock_requests_response(502, "Bad gateway")
self.assertRaises(requests.exceptions.HTTPError, self.nailgun.get_tasks)
def test_put_deployment_tasks(self):
release = {"id": "1"}
tasks = []
self.mock_request.put.return_value = self.mock_requests_response(200, "[]")
response = self.nailgun.put_deployment_tasks(release, tasks)
self.assertEqual(response, tasks)
self.mock_request.put.return_value = self.mock_requests_response(502, "Bad gateway")
self.assertRaises(requests.exceptions.HTTPError, self.nailgun.put_deployment_tasks, release, tasks)
class OpenStackUpgrader(UpgradeEngine):
"""OpenStack Upgrader.
The class is designed to do the following tasks:
* install manifests in the system
* add new releases to nailgun's database
* add notification about new releases
"""
def __init__(self, *args, **kwargs):
super(OpenStackUpgrader, self).__init__(*args, **kwargs)
#: a list of releases to install
self.releases = self._read_releases()
#: a nailgun object - api wrapper
self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])
self._reset_state()
def upgrade(self):
self._reset_state()
self.install_puppets()
self.install_releases()
self.install_versions()
def rollback(self):
self.remove_releases()
self.remove_puppets()
self.remove_versions()
def install_puppets(self):
logger.info('Installing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_puppets(self):
logger.info('Removing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(
self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.remove(destination)
def install_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
utils.create_dir_if_not_exists(release_versions_cfg['dst'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.copy(version_file, dst)
def remove_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
for version_file in versions:
dst = os.path.join(
release_versions_cfg['dst'],
os.path.basename(version_file))
utils.remove(dst)
def install_releases(self):
# add only new releases to nailgun and inject paths to
# base repo if needed
existing_releases = self.nailgun.get_releases()
releases = self._get_unique_releases(self.releases, existing_releases)
# upload unexisting releases
for release in releases:
# register new release
logger.debug('Register a new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_release(release)
# save release id for futher possible rollback
self._rollback_ids['release'].append(response['id'])
self.upload_release_deployment_tasks(response)
# add notification abot successfull releases
logger.debug('Add notification about new release: %s (%s)',
release['name'],
release['version'])
response = self.nailgun.create_notification({
'topic': 'release',
'message': 'New release available: {0} ({1})'.format(
release['name'],
release['version'],
),
})
# save notification id for futher possible rollback
self._rollback_ids['notification'].append(response['id'])
def upload_release_deployment_tasks(self, release):
"""Performs os.walk by puppet src, matches all files with tasks
of given pattern and uploads this for release.
:param release: dict representation of release
"""
tasks = []
release_puppet_path = os.path.join(
self.config.openstack['puppets']['dst'], release['version'])
for file_path in utils.iterfiles_filter(
release_puppet_path,
self.config.deployment_tasks_file_pattern):
tasks.extend(utils.read_from_yaml(file_path))
self.nailgun.put_deployment_tasks(release, tasks)
def remove_releases(self):
"""Remove all releases that are created by current session.
"""
for release_id in reversed(self._rollback_ids['release']):
try:
logger.debug('Removing release with ID=%s', release_id)
self.nailgun.remove_release(release_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception('%s', six.text_type(exc))
for notif_id in reversed(self._rollback_ids['notification']):
try:
logger.debug('Removing notification with ID=%s', notif_id)
self.nailgun.remove_notification(notif_id)
except (
requests.exceptions.HTTPError
) as exc:
logger.exception('%s', six.text_type(exc))
def _reset_state(self):
"""Remove rollback IDs from the arrays.
"""
#: a list of ids that have to be removed in case of rollback
self._rollback_ids = {
'release': [],
'notification': [],
}
@classmethod
def _get_unique_releases(cls, releases, existing_releases):
"""Returns a list of releases that aren't exist yet.
:param releases: a list of releases to filter
:param existing_releases: a list of existing releases
:returns: a list of unique releases
"""
existing_releases = [
(r['name'], r['version']) for r in existing_releases
]
unique = lambda r: (r['name'], r['version']) not in existing_releases
return [r for r in releases if unique(r)]
def _read_releases(self):
"""Returns a list of releases in a dict representation.
"""
releases = []
# read releases from a set of files
for release_yaml in glob.glob(self.config.openstack['releases']):
with io.open(release_yaml, 'r', encoding='utf-8') as f:
releases.extend(utils.load_fixture(f))
return releases
@property
def required_free_space(self):
spaces = {
self.config.openstack['puppets']['dst']:
glob.glob(self.config.openstack['puppets']['src']), }
for dst, srcs in six.iteritems(spaces):
size = 0
for src in srcs:
size += utils.dir_size(src)
spaces[dst] = size
return spaces
class TestNailgunClient(base.BaseTestCase):
def setUp(self):
mock_keystone = mock.MagicMock()
self.mock_request = mock_keystone.request
with mock.patch(
'fuel_upgrade.clients.nailgun_client.KeystoneClient',
return_value=mock_keystone):
self.nailgun = NailgunClient('127.0.0.1', 8000)
def test_create_release(self):
# test normal bahavior
self.mock_request.post.return_value = self.mock_requests_response(
201, '{ "id": "42" }')
response = self.nailgun.create_release({
'name': 'Havana on Ubuntu 12.04'})
self.assertEqual(response, {'id': '42'})
# test failed result
self.mock_request.post.return_value.status_code = 409
self.assertRaises(
requests.exceptions.HTTPError,
self.nailgun.create_release,
{'name': 'Havana on Ubuntu 12.04'})
def test_delete_release(self):
# test normal bahavior
for status in (200, 204):
self.mock_request.delete.return_value = \
self.mock_requests_response(status, 'No Content')
response = self.nailgun.remove_release(42)
self.assertEqual(response, 'No Content')
# test failed result
self.mock_request.delete.return_value = self.mock_requests_response(
409, 'Conflict')
self.assertRaises(
requests.exceptions.HTTPError,
self.nailgun.remove_release,
42)
def test_create_notification(self):
# test normal bahavior
self.mock_request.post.return_value = self.mock_requests_response(
201,
'{ "id": "42" }')
response = self.nailgun.create_notification({
'topic': 'release',
'message': 'New release available!'})
self.assertEqual(response, {'id': '42'})
# test failed result
self.mock_request.post.return_value.status_code = 409
self.assertRaises(
requests.exceptions.HTTPError,
self.nailgun.create_notification,
{'topic': 'release',
'message': 'New release available!'})
def test_delete_notification(self):
# test normal bahavior
for status in (200, 204):
self.mock_request.delete.return_value = \
self.mock_requests_response(status, 'No Content')
response = self.nailgun.remove_notification(42)
self.assertEqual(response, 'No Content')
# test failed result
self.mock_request.delete.return_value = self.mock_requests_response(
409, 'Conflict')
self.assertRaises(
requests.exceptions.HTTPError,
self.nailgun.remove_notification,
42)
def test_get_tasks(self):
# test positive cases
self.mock_request.get.return_value = self.mock_requests_response(
200, '[1,2,3]')
response = self.nailgun.get_tasks()
self.assertEqual(response, [1, 2, 3])
# test negative cases
self.mock_request.get.return_value = self.mock_requests_response(
502, 'Bad gateway')
self.assertRaises(
requests.exceptions.HTTPError, self.nailgun.get_tasks)
class OpenStackUpgrader(UpgradeEngine):
"""OpenStack Upgrader.
The class is designed to do the following tasks:
* install repos in the system
* install manifests in the system
* add new releases to nailgun's database
* add notification about new releases
"""
def __init__(self, *args, **kwargs):
super(OpenStackUpgrader, self).__init__(*args, **kwargs)
#: a list of releases to install
self.releases = self._read_releases()
#: a nailgun object - api wrapper
self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])
self._reset_state()
def upgrade(self):
self._reset_state()
self.install_puppets()
self.install_repos()
self.install_releases()
self.install_versions()
def rollback(self):
self.remove_releases()
self.remove_repos()
self.remove_puppets()
self.remove_versions()
def install_puppets(self):
logger.info('Installing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_puppets(self):
logger.info('Removing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.remove(destination)
def install_repos(self):
logger.info('Installing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_repos(self):
logger.info('Removing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.remove(destination)
def on_success(self):
"""Do nothing for this engine
"""
def install_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
utils.create_dir_if_not_exists(release_versions_cfg['dst'])
for version_file in versions:
dst = os.path.join(release_versions_cfg['dst'],
os.path.basename(version_file))
utils.copy(version_file, dst)
def remove_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
for version_file in versions:
dst = os.path.join(release_versions_cfg['dst'],
os.path.basename(version_file))
utils.remove(dst)
def install_releases(self):
# check releases for existing in nailgun side
releases = self._get_unique_releases(self.releases,
self.nailgun.get_releases())
# upload unexisting releases
for release in releases:
# register new release
logger.debug('Register a new release: %s (%s)', release['name'],
release['version'])
response = self.nailgun.create_release(release)
# save release id for futher possible rollback
self._rollback_ids['release'].append(response['id'])
# add notification abot successfull releases
logger.debug('Add notification about new release: %s (%s)',
release['name'], release['version'])
response = self.nailgun.create_notification({
'topic':
'release',
'message':
'New release available: {0} ({1})'.format(
release['name'],
release['version'],
),
})
# save notification id for futher possible rollback
self._rollback_ids['notification'].append(response['id'])
def remove_releases(self):
"""Remove all releases that are created by current session.
"""
for release_id in reversed(self._rollback_ids['release']):
try:
logger.debug('Removing release with ID=%s', release_id)
self.nailgun.remove_release(release_id)
except (requests.exceptions.HTTPError) as exc:
logger.exception(six.text_type(exc))
for notif_id in reversed(self._rollback_ids['notification']):
try:
logger.debug('Removing notification with ID=%s', notif_id)
self.nailgun.remove_notification(notif_id)
except (requests.exceptions.HTTPError) as exc:
logger.exception(six.text_type(exc))
def _reset_state(self):
"""Remove rollback IDs from the arrays.
"""
#: a list of ids that have to be removed in case of rollback
self._rollback_ids = {
'release': [],
'notification': [],
}
@classmethod
def _get_unique_releases(cls, releases, existing_releases):
"""Returns a list of releases that aren't exist yet.
:param releases: a list of releases to filter
:param existing_releases: a list of existing releases
:returns: a list of unique releases
"""
existing_releases = [(r['name'], r['version'])
for r in existing_releases]
unique = lambda r: (r['name'], r['version']) not in existing_releases
return [r for r in releases if unique(r)]
def _read_releases(self):
"""Returns a list of releases in a dict representation.
"""
releases = []
# read releases from a set of files
for release_yaml in glob.glob(self.config.openstack['releases']):
with io.open(release_yaml, 'r', encoding='utf-8') as f:
releases.extend(utils.load_fixture(f))
# inject orchestrator_data into releases if empty
for release in releases:
meta = {
'version': release['version'],
'master_ip': self.config.astute['ADMIN_NETWORK']['ipaddress']
}
if 'ubuntu' == release['operating_system'].lower():
repo = 'http://{master_ip}:8080/{version}/ubuntu/x86_64 ' \
'precise main'
else:
repo = 'http://{master_ip}:8080/{version}/centos/x86_64'
release['orchestrator_data'] = {
'puppet_manifests_source':
('rsync://{master_ip}:/puppet/{version}/manifests/'.format(
**meta)),
'puppet_modules_source':
('rsync://{master_ip}:/puppet/{version}/modules/'.format(
**meta)),
'repo_metadata': {
'nailgun': repo.format(**meta)
}
}
return releases
@property
def required_free_space(self):
spaces = {
self.config.openstack['puppets']['dst']:
glob.glob(self.config.openstack['puppets']['src']),
self.config.openstack['repos']['dst']:
glob.glob(self.config.openstack['repos']['src'])
}
for dst, srcs in six.iteritems(spaces):
size = 0
for src in srcs:
size += utils.dir_size(src)
spaces[dst] = size
return spaces
class OpenStackUpgrader(UpgradeEngine):
"""OpenStack Upgrader.
The class is designed to do the following tasks:
* install repos in the system
* install manifests in the system
* add new releases to nailgun's database
* add notification about new releases
"""
def __init__(self, *args, **kwargs):
super(OpenStackUpgrader, self).__init__(*args, **kwargs)
#: a list of releases to install
self.releases = self._read_releases()
#: a nailgun object - api wrapper
self.nailgun = NailgunClient(**self.config.endpoints['nginx_nailgun'])
self._reset_state()
def upgrade(self):
self._reset_state()
self.install_puppets()
self.install_repos()
self.install_releases()
self.install_versions()
def rollback(self):
self.remove_releases()
self.remove_repos()
self.remove_puppets()
self.remove_versions()
def install_puppets(self):
logger.info('Installing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_puppets(self):
logger.info('Removing puppet manifests...')
sources = glob.glob(self.config.openstack['puppets']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['puppets']['dst'],
os.path.basename(source))
utils.remove(destination)
def install_repos(self):
logger.info('Installing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.copy(source, destination)
def remove_repos(self):
logger.info('Removing repositories...')
sources = glob.glob(self.config.openstack['repos']['src'])
for source in sources:
destination = os.path.join(self.config.openstack['repos']['dst'],
os.path.basename(source))
utils.remove(destination)
def on_success(self):
"""Do nothing for this engine
"""
def install_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
utils.create_dir_if_not_exists(release_versions_cfg['dst'])
for version_file in versions:
dst = os.path.join(release_versions_cfg['dst'],
os.path.basename(version_file))
utils.copy(version_file, dst)
def remove_versions(self):
"""Copy openstack release versions
"""
logger.info('Copy openstack release versions...')
release_versions_cfg = self.config.openstack['release_versions']
versions = glob.glob(release_versions_cfg['src'])
for version_file in versions:
dst = os.path.join(release_versions_cfg['dst'],
os.path.basename(version_file))
utils.remove(dst)
def install_releases(self):
# add only new releases to nailgun and inject paths to
# base repo if needed
existing_releases = self.nailgun.get_releases()
releases = self._get_unique_releases(self.releases, existing_releases)
self._add_base_repos_to_releases(releases, existing_releases)
# upload unexisting releases
for release in releases:
# register new release
logger.debug('Register a new release: %s (%s)', release['name'],
release['version'])
response = self.nailgun.create_release(release)
# save release id for futher possible rollback
self._rollback_ids['release'].append(response['id'])
# add notification abot successfull releases
logger.debug('Add notification about new release: %s (%s)',
release['name'], release['version'])
response = self.nailgun.create_notification({
'topic':
'release',
'message':
'New release available: {0} ({1})'.format(
release['name'],
release['version'],
),
})
# save notification id for futher possible rollback
self._rollback_ids['notification'].append(response['id'])
def remove_releases(self):
"""Remove all releases that are created by current session.
"""
for release_id in reversed(self._rollback_ids['release']):
try:
logger.debug('Removing release with ID=%s', release_id)
self.nailgun.remove_release(release_id)
except (requests.exceptions.HTTPError) as exc:
logger.exception('%s', six.text_type(exc))
for notif_id in reversed(self._rollback_ids['notification']):
try:
logger.debug('Removing notification with ID=%s', notif_id)
self.nailgun.remove_notification(notif_id)
except (requests.exceptions.HTTPError) as exc:
logger.exception('%s', six.text_type(exc))
def _reset_state(self):
"""Remove rollback IDs from the arrays.
"""
#: a list of ids that have to be removed in case of rollback
self._rollback_ids = {
'release': [],
'notification': [],
}
@classmethod
def _get_unique_releases(cls, releases, existing_releases):
"""Returns a list of releases that aren't exist yet.
:param releases: a list of releases to filter
:param existing_releases: a list of existing releases
:returns: a list of unique releases
"""
existing_releases = [(r['name'], r['version'])
for r in existing_releases]
unique = lambda r: (r['name'], r['version']) not in existing_releases
return [r for r in releases if unique(r)]
def _read_releases(self):
"""Returns a list of releases in a dict representation.
"""
releases = []
# read releases from a set of files
for release_yaml in glob.glob(self.config.openstack['releases']):
with io.open(release_yaml, 'r', encoding='utf-8') as f:
releases.extend(utils.load_fixture(f))
# inject orchestrator_data into releases if empty
#
# NOTE(ikalnitsky): we can drop this block of code when
# we got two things done:
# * remove `fuelweb` word from default repos
# * add this template to `openstack.yaml`
# * fill orchestrator_data in nailgun during syncdb
for release in releases:
repo_path = \
'http://{{MASTER_IP}}:8080/{{OPENSTACK_VERSION}}/{OS}/x86_64'\
.format(OS=release['operating_system'].lower())
if release['operating_system'].lower() == 'ubuntu':
repo_path += ' precise main'
release['orchestrator_data'] = {
'puppet_manifests_source':
'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/',
'puppet_modules_source':
'rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/',
'repo_metadata': {
release['version']: repo_path
}
}
return releases
def _add_base_repos_to_releases(self, releases, existing_releases):
"""Update given releases with orchestrator data of base release.
:param releases: a list of releases to process
:param existing_releases: a list of existings releases
"""
metadata_path = self.config.openstack['metadata']
# do nothing in case of metadata.yaml absence - just assume
# that we have full repos
if not os.path.exists(metadata_path):
return
with io.open(metadata_path, 'r', encoding='utf-8') as f:
metadata = yaml.load(f) or {}
# keep diff-based releases
releases = filter(
lambda r: r['version'] in metadata.get('diff_releases', {}),
releases)
# inject repos from base releases
for release in releases:
version = release['version']
base_release = utils.get_base_release(
release, metadata['diff_releases'][version], existing_releases)
if base_release is None:
raise errors.BaseReleaseNotFound(
'Could not find a base release - "{0}" - of the '
'release "{1}".'.format(metadata['diff_releases'][version],
version))
release['orchestrator_data']['repo_metadata'].update(
base_release['orchestrator_data']['repo_metadata'])
@property
def required_free_space(self):
spaces = {
self.config.openstack['puppets']['dst']:
glob.glob(self.config.openstack['puppets']['src']),
self.config.openstack['repos']['dst']:
glob.glob(self.config.openstack['repos']['src'])
}
for dst, srcs in six.iteritems(spaces):
size = 0
for src in srcs:
size += utils.dir_size(src)
spaces[dst] = size
return spaces