def __init__(self, upgraders, config, no_rollback=True):
#: an object with configuration context
self._config = config
#: a list of upgraders to use
self._upgraders = upgraders
#: a list of used upgraders (needs by rollback feature)
self._used_upgraders = []
#: should we make rollback in case of error?
self._rollback = not no_rollback
#: version.yaml manager
self._version_file = VersionFile(self._config)
self._version_file.save_current()
def __init__(self, *args, **kwargs):
super(HostSystemUpgrader, self).__init__(*args, **kwargs)
#: host system upgarder specific configs
self.host_system_config = self.config.host_system
#: path to puppet manifests
self.manifest_path = self.host_system_config['manifest_path']
#: path to puppet modules
self.puppet_modules_path = self.host_system_config[
'puppet_modules_path']
#: path to repo template
self.repo_template_path = os.path.join(
self.templates_dir, 'nailgun.repo')
#: new version of fuel
self.version = self.config.new_version
#: path to repository config
self.repo_config_path = self.host_system_config['repo_config_path']
#: path to local repository
self.repo_path = self.host_system_config['repo_path']
#: version file manager
self.version_file = VersionFile(self.config)
self.version_file.save_current()
def __init__(self, upgraders, config, no_rollback=True):
#: an object with configuration context
self._config = config
#: a list of upgraders to use
self._upgraders = upgraders
#: a list of used upgraders (needs by rollback feature)
self._used_upgraders = []
#: should we make rollback in case of error?
self._rollback = not no_rollback
#: version.yaml manager
self._version_file = VersionFile(self._config)
self._version_file.save_current()
def __init__(self, *args, **kwargs):
super(DockerUpgrader, self).__init__(*args, **kwargs)
self.working_directory = self.config.working_directory
utils.create_dir_if_not_exists(self.working_directory)
self.docker_client = docker.Client(
base_url=self.config.docker['url'],
version=self.config.docker['api_version'],
timeout=self.config.docker['http_timeout'])
self.new_release_containers = self.make_new_release_containers_list()
self.cobbler_config_path = self.config.cobbler_config_path.format(
working_directory=self.working_directory)
self.upgrade_verifier = FuelUpgradeVerify(self.config)
self.from_version = self.config.from_version
self.supervisor = SupervisorClient(self.config, self.from_version)
self.version_file = VersionFile(self.config)
self.version_file.save_current()
class TestVersionFile(BaseTestCase):
def setUp(self):
self.version_file = VersionFile(self.fake_config)
@mock.patch('fuel_upgrade.version_file.utils')
def test_save_current(self, mock_utils):
self.version_file.save_current()
mock_utils.copy_if_does_not_exist.assert_called_once_with(
'/etc/fuel/version.yaml',
'/var/lib/fuel_upgrade/9999/version.yaml')
@mock.patch('fuel_upgrade.version_file.utils')
def test_switch_to_new(self, mock_utils):
self.version_file.switch_to_new()
mock_utils.create_dir_if_not_exists.assert_called_once_with(
'/etc/fuel/9999')
mock_utils.copy.assert_called_once_with(
'/tmp/upgrade_path/config/version.yaml',
'/etc/fuel/9999/version.yaml')
mock_utils.symlink.assert_called_once_with(
'/etc/fuel/9999/version.yaml',
'/etc/fuel/version.yaml')
@mock.patch('fuel_upgrade.version_file.utils.symlink')
def test_switch_to_previous(self, symlink_mock):
self.version_file.switch_to_previous()
symlink_mock.assert_called_once_with(
'/etc/fuel/0/version.yaml',
'/etc/fuel/version.yaml')
class TestVersionFile(BaseTestCase):
def setUp(self):
self.version_file = VersionFile(self.fake_config)
@mock.patch('fuel_upgrade.version_file.utils')
def test_save_current(self, mock_utils):
self.version_file.save_current()
mock_utils.copy_if_does_not_exist.assert_called_once_with(
'/etc/fuel/version.yaml',
'/var/lib/fuel_upgrade/9999/version.yaml')
@mock.patch('fuel_upgrade.version_file.utils')
def test_switch_to_new(self, mock_utils):
self.version_file.switch_to_new()
mock_utils.create_dir_if_not_exists.assert_called_once_with(
'/etc/fuel/9999')
mock_utils.copy.assert_called_once_with(
'/tmp/upgrade_path/config/version.yaml',
'/etc/fuel/9999/version.yaml')
mock_utils.symlink.assert_called_once_with(
'/etc/fuel/9999/version.yaml', '/etc/fuel/version.yaml')
@mock.patch('fuel_upgrade.version_file.utils.symlink')
def test_switch_to_previous(self, symlink_mock):
self.version_file.switch_to_previous()
symlink_mock.assert_called_once_with('/etc/fuel/0/version.yaml',
'/etc/fuel/version.yaml')
def __init__(self, *args, **kwargs):
super(DockerUpgrader, self).__init__(*args, **kwargs)
self.working_directory = self.config.working_directory
utils.create_dir_if_not_exists(self.working_directory)
self.docker_client = docker.Client(
base_url=self.config.docker['url'],
version=self.config.docker['api_version'],
timeout=self.config.docker['http_timeout'])
self.new_release_containers = self.make_new_release_containers_list()
self.cobbler_config_path = self.config.cobbler_config_path.format(
working_directory=self.working_directory)
self.upgrade_verifier = FuelUpgradeVerify(self.config)
self.from_version = self.config.from_version
self.supervisor = SupervisorClient(self.config, self.from_version)
self.version_file = VersionFile(self.config)
self.version_file.save_current()
class UpgradeManager(object):
"""Upgrade manager is used to orchestrate upgrading process.
:param upgraders: a list with upgrader classes to use; each upgrader
must inherit the :class:`BaseUpgrader`
:param no_rollback: call :meth:`BaseUpgrader.rollback` method
in case of exception during execution
"""
def __init__(self, upgraders, config, no_rollback=True):
#: an object with configuration context
self._config = config
#: a list of upgraders to use
self._upgraders = upgraders
#: a list of used upgraders (needs by rollback feature)
self._used_upgraders = []
#: should we make rollback in case of error?
self._rollback = not no_rollback
#: version.yaml manager
self._version_file = VersionFile(self._config)
self._version_file.save_current()
def run(self):
"""Runs consequentially all registered upgraders.
.. note:: in case of exception the `rollback` method will be called
"""
logger.info('*** START UPGRADING')
self._version_file.switch_to_new()
for upgrader in self._upgraders:
logger.debug('%s: backuping...', upgrader.__class__.__name__)
try:
upgrader.backup()
except Exception as exc:
logger.exception(
'%s: failed to backup: "%s"',
upgrader.__class__.__name__, exc)
logger.error('*** UPGRADE FAILED')
raise
for upgrader in self._upgraders:
logger.debug('%s: upgrading...', upgrader.__class__.__name__)
self._used_upgraders.append(upgrader)
try:
upgrader.upgrade()
except Exception as exc:
logger.exception(
'%s: failed to upgrade: "%s"',
upgrader.__class__.__name__, exc)
if self._rollback:
self.rollback()
logger.error('*** UPGRADE FAILED')
raise
try:
self._on_success()
except Exception as exc:
logger.exception(
'Could not complete on_success actions due to %s',
six.text_type(exc))
logger.info('*** UPGRADING MASTER NODE DONE SUCCESSFULLY')
logger.info('*** PLEASE REBOOT YOUR BOOTSTRAP NODES IN ORDER TO MAKE'
' SURE THAT THEY USE THE LATEST BOOTSTRAP IMAGE')
def _on_success(self):
"""Do some useful job if upgrade was done successfully.
"""
# Remove saved version files for all upgrades
#
# NOTE(eli): It solves several problems:
#
# 1. user runs upgrade 5.0 -> 5.1 which fails
# upgrade system saves version which we upgrade
# from in file working_dir/5.1/version.yaml.
# Then user runs upgrade 5.0 -> 5.0.1 which
# successfully upgraded. Then user runs again
# upgrade 5.0.1 -> 5.1, but there is saved file
# working_dir/5.1/version.yaml which contains
# 5.0 version, and upgrade system thinks that
# it's upgrading from 5.0 version, as result
# it tries to make database dump from wrong
# version of container.
#
# 2. without this hack user can run upgrade
# second time and loose his data, this hack
# prevents this case because before upgrade
# checker will use current version instead
# of saved version to determine version which
# we run upgrade from.
for version_file in glob.glob(self._config.version_files_mask):
utils.remove(version_file)
def rollback(self):
logger.debug('Run rollback')
# because of issue #1452378 [1], we have to perform HostSystem's
# rollback before others. so, move it to the end of list.
#
# [1]: https://bugs.launchpad.net/fuel/+bug/1452378
hostsystem = next((
upgrader for upgrader in self._used_upgraders
if isinstance(upgrader, HostSystemUpgrader)),
None)
if hostsystem is not None:
self._used_upgraders.remove(hostsystem)
self._used_upgraders.append(hostsystem)
# do rollback in reverse order for all the upgraders
while self._used_upgraders:
upgrader = self._used_upgraders.pop()
logger.debug('%s: rollbacking...', upgrader.__class__.__name__)
upgrader.rollback()
self._version_file.switch_to_previous()
class DockerUpgrader(UpgradeEngine):
"""Docker management system for upgrades
"""
def __init__(self, *args, **kwargs):
super(DockerUpgrader, self).__init__(*args, **kwargs)
self.working_directory = self.config.working_directory
utils.create_dir_if_not_exists(self.working_directory)
self.docker_client = docker.Client(
base_url=self.config.docker['url'],
version=self.config.docker['api_version'],
timeout=self.config.docker['http_timeout'])
self.new_release_images = self.make_new_release_images_list()
self.new_release_containers = self.make_new_release_containers_list()
self.cobbler_config_path = self.config.cobbler_config_path.format(
working_directory=self.working_directory)
self.upgrade_verifier = FuelUpgradeVerify(self.config)
self.from_version = self.config.from_version
self.supervisor = SupervisorClient(self.config, self.from_version)
self.version_file = VersionFile(self.config)
self.version_file.save_current()
def upgrade(self):
"""Method with upgarde logic
"""
# Preapre env for upgarde
self.save_db()
self.save_cobbler_configs()
# Run upgrade
self.supervisor.stop_all_services()
self.stop_fuel_containers()
self.upload_images()
self.stop_fuel_containers()
self.create_containers()
self.stop_fuel_containers()
# Update configs and run new services
self.generate_configs()
self.switch_to_new_configs()
self.version_file.switch_to_new()
self.supervisor.restart_and_wait()
self.upgrade_verifier.verify()
def rollback(self):
"""Method which contains rollback logic
"""
self.version_file.switch_to_previous()
self.supervisor.switch_to_previous_configs()
self.supervisor.stop_all_services()
self.stop_fuel_containers()
self.supervisor.restart_and_wait()
def on_success(self):
"""Remove saved version files for all upgrades
NOTE(eli): It solves several problems:
1. user runs upgrade 5.0 -> 5.1 which fails
upgrade system saves version which we upgrade
from in file working_dir/5.1/version.yaml.
Then user runs upgrade 5.0 -> 5.0.1 which
successfully upgraded. Then user runs again
upgrade 5.0.1 -> 5.1, but there is saved file
working_dir/5.1/version.yaml which contains
5.0 version, and upgrade system thinks that
it's upgrading from 5.0 version, as result
it tries to make database dump from wrong
version of container.
2. without this hack user can run upgrade
second time and loose his data, this hack
prevents this case because before upgrade
checker will use current version instead
of saved version to determine version which
we run upgrade from.
"""
for version_file in glob.glob(self.config.version_files_mask):
utils.remove(version_file)
@property
def required_free_space(self):
"""Required free space to run upgrade
* space for docker
* several megabytes for configs
* reserve several megabytes for working directory
where we keep postgresql dump and cobbler configs
:returns: dict where key is path to directory
and value is required free space
"""
return {
self.config.docker['dir']: self._calculate_images_size(),
self.config.supervisor['configs_prefix']: 10,
self.config.fuel_config_path: 10,
self.working_directory: 150}
def _calculate_images_size(self):
images_list = [i['docker_image'] for i in self.new_release_images]
return utils.files_size(images_list)
def save_db(self):
"""Saves postgresql database into the file
"""
logger.debug(u'Backup database')
pg_dump_path = os.path.join(self.working_directory, 'pg_dump_all.sql')
pg_dump_files = utils.VersionedFile(pg_dump_path)
pg_dump_tmp_path = pg_dump_files.next_file_name()
try:
container_name = self.make_container_name(
'postgres', self.from_version)
self.exec_cmd_in_container(
container_name,
u"su postgres -c 'pg_dumpall --clean' > {0}".format(
pg_dump_tmp_path))
except (errors.ExecutedErrorNonZeroExitCode,
errors.CannotFindContainerError) as exc:
utils.remove_if_exists(pg_dump_tmp_path)
if not utils.file_exists(pg_dump_path):
raise
logger.debug(
u'Failed to make database dump, '
'will be used dump from previous run: %s', exc)
valid_dumps = filter(utils.verify_postgres_dump,
pg_dump_files.sorted_files())
if valid_dumps:
utils.hardlink(valid_dumps[0], pg_dump_path, overwrite=True)
map(utils.remove_if_exists,
valid_dumps[self.config.keep_db_backups_count:])
else:
raise errors.DatabaseDumpError(
u'Failed to make database dump, there '
'are no valid database backup '
'files, {0}'.format(pg_dump_path))
def save_cobbler_configs(self):
"""Copy config files from container
"""
container_name = self.make_container_name(
'cobbler', self.from_version)
try:
utils.exec_cmd('docker cp {0}:{1} {2}'.format(
container_name,
self.config.cobbler_container_config_path,
self.cobbler_config_path))
except errors.ExecutedErrorNonZeroExitCode:
utils.rmtree(self.cobbler_config_path)
raise
self.verify_cobbler_configs()
def verify_cobbler_configs(self):
"""Verify that cobbler config directory
contains valid data
"""
configs = glob.glob(
self.config.cobbler_config_files_for_verifier.format(
cobbler_config_path=self.cobbler_config_path))
# NOTE(eli): cobbler config directory should
# contain at least one file (default.json)
if len(configs) < 1:
raise errors.WrongCobblerConfigsError(
u'Cannot find json files in directory {0}'.format(
self.cobbler_config_path))
for config in configs:
if not utils.check_file_is_valid_json(config):
raise errors.WrongCobblerConfigsError(
u'Invalid json config {0}'.format(config))
def upload_images(self):
"""Uploads images to docker
"""
logger.info(u'Start image uploading')
for image in self.new_release_images:
logger.debug(u'Try to upload docker image {0}'.format(image))
docker_image = image['docker_image']
if not os.path.exists(docker_image):
logger.warn(u'Cannot find docker image "{0}"'.format(
docker_image))
continue
# NOTE(eli): docker-py binding
# doesn't have equal call for
# image importing which equals to
# `docker load`
utils.exec_cmd(u'docker load < "{0}"'.format(docker_image))
def create_containers(self):
"""Create containers in the right order
"""
logger.info(u'Started containers creation')
graph = self.build_dependencies_graph(self.new_release_containers)
logger.debug(u'Built dependencies graph {0}'.format(graph))
containers_to_creation = utils.topological_sorting(graph)
logger.debug(u'Resolved creation order {0}'.format(
containers_to_creation))
for container_id in containers_to_creation:
container = self.container_by_id(container_id)
logger.debug(u'Start container {0}'.format(container))
links = self.get_container_links(container)
created_container = self.create_container(
container['image_name'],
name=container.get('container_name'),
volumes=container.get('volumes'),
ports=container.get('ports'),
detach=False)
volumes_from = []
for container_id in container.get('volumes_from', []):
volume_container = self.container_by_id(container_id)
volumes_from.append(volume_container['container_name'])
self.start_container(
created_container,
port_bindings=container.get('port_bindings'),
links=links,
volumes_from=volumes_from,
binds=container.get('binds'),
privileged=container.get('privileged', False))
if container.get('after_container_creation_command'):
self.run_after_container_creation_command(container)
def run_after_container_creation_command(self, container):
"""Runs command in container with retries in
case of error
:param container: dict with container information
"""
command = container['after_container_creation_command']
def execute():
self.exec_cmd_in_container(container['container_name'], command)
self.exec_with_retries(
execute, errors.ExecutedErrorNonZeroExitCode,
'', retries=30, interval=4)
def exec_cmd_in_container(self, container_name, cmd):
"""Execute command in running container
:param name: name of the container, like fuel-core-5.1-nailgun
"""
db_container_id = self.container_docker_id(container_name)
# NOTE(eli): we don't use dockerctl shell
# instead of lxc-attach here because
# release 5.0 has a bug which doesn't
# allow us to use quotes in command
# https://bugs.launchpad.net/fuel/+bug/1324200
utils.exec_cmd(
"lxc-attach --name {0} -- {1}".format(
db_container_id, cmd))
def get_ports(self, container):
"""Docker binding accepts ports as tuple,
here we convert from list to tuple.
FIXME(eli): https://github.com/dotcloud/docker-py/blob/
73434476b32136b136e1cdb0913fd123126f2a52/
docker/client.py#L111-L114
"""
ports = container.get('ports')
if ports is None:
return
return [port if not isinstance(port, list) else tuple(port)
for port in ports]
def exec_with_retries(
self, func, exceptions, message, retries=0, interval=0):
# TODO(eli): refactor it and make retries
# as a decorator
intervals = retries * [interval]
for interval in intervals:
try:
return func()
except exceptions as exc:
if str(exc).endswith(message):
time.sleep(interval)
continue
raise
return func()
def get_container_links(self, container):
links = []
if container.get('links'):
for container_link in container.get('links'):
link_container = self.container_by_id(
container_link['id'])
links.append((
link_container['container_name'],
container_link['alias']))
return links
@classmethod
def build_dependencies_graph(cls, containers):
"""Builds graph which based on
`volumes_from` and `link` parameters
of container.
:returns: dict where keys are nodes and
values are lists of dependencies
"""
graph = {}
for container in containers:
graph[container['id']] = sorted(set(
container.get('volumes_from', []) +
[link['id'] for link in container.get('links', [])]))
return graph
def generate_configs(self):
"""Generates supervisor configs
and saves them to configs directory
"""
configs = []
for container in self.new_release_containers:
params = {
'service_name': container['id'],
'command': u'docker start -a {0}'.format(
container['container_name'])
}
if container['supervisor_config']:
configs.append(params)
self.supervisor.generate_configs(configs)
cobbler_container = self.container_by_id('cobbler')
self.supervisor.generate_cobbler_config(
{'service_name': cobbler_container['id'],
'container_name': cobbler_container['container_name']})
def switch_to_new_configs(self):
"""Switches supervisor to new configs
"""
self.supervisor.switch_to_new_configs()
def build_images(self):
"""Use docker API to build new containers
"""
self.remove_new_release_images()
for image in self.new_release_images:
logger.info(u'Start image building: {0}'.format(image))
self.docker_client.build(
path=image['docker_file'],
tag=image['name'],
nocache=True)
# NOTE(eli): 0.10 and early versions of
# Docker api dont't return correct http
# response in case of failed build, here
# we check if build succed and raise error
# if it failed i.e. image was not created
if not self.docker_client.images(name=image):
raise errors.DockerFailedToBuildImageError(
u'Failed to build image {0}'.format(image))
def volumes_dependencies(self, container):
"""Get list of `volumes` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'volumes_from')
def link_dependencies(self, container):
"""Get list of `link` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'link')
def dependencies_names(self, container, key):
"""Returns list of dependencies for specified key
:param contaienr: dict with information about container
:param key: key which will be used for dependencies retrieving
:returns: list of container names
"""
names = []
if container.get(key):
for container_id in container.get(key):
container = self.container_by_id(container_id)
names.append(container['container_name'])
return names
def stop_fuel_containers(self):
"""Use docker API to shutdown containers
"""
containers = self.docker_client.containers(limit=-1)
containers_to_stop = filter(
lambda c: c['Image'].startswith(self.config.image_prefix),
containers)
for container in containers_to_stop:
logger.debug(u'Stop container: {0}'.format(container))
self.stop_container(container['Id'])
public_ports = self._get_docker_container_public_ports(
containers_to_stop)
self.clean_docker_iptables_rules(public_ports)
def _get_docker_container_public_ports(self, containers):
"""Returns public ports
:param containers: list of dicts with information about
containers which have `Ports` list
with items where exist `PublicPort`
field
:returns: list of public ports
"""
container_ports = []
for container in containers:
container_ports.extend(container['Ports'])
return [container_port['PublicPort']
for container_port in container_ports]
def clean_docker_iptables_rules(self, ports):
"""Sometimes when we run docker stop
(version dc9c28f/0.10.0) it doesn't clean
iptables rules, as result when we run new
container on the same port we have two rules
with the same port but with different IPs,
we have to clean this rules to prevent services
unavailability.
Example of the problem:
$ iptables -t nat -S
...
-A DOCKER -p tcp -m tcp --dport 443 -j DNAT \
--to-destination 172.17.0.7:443
-A DOCKER -p tcp -m tcp --dport 443 -j DNAT \
--to-destination 172.17.0.3:443
:param ports: list of ports to clean up
"""
rules_to_deletion = []
patterns = [re.compile(
'^-A DOCKER .+ --dport {0} '
'-j DNAT --to-destination .+'.format(port)) for port in ports]
for rule in utils.exec_cmd_iterator('iptables -t nat -S'):
for pattern in patterns:
if pattern.match(rule):
rules_to_deletion.append(rule)
for rule in rules_to_deletion:
# Remove -A (add) prefix and use -D (delete) instead
utils.exec_cmd('iptables -t nat -D {0}'.format(rule[2:]))
# NOTE(eli): Run list of rules again,
# it's required to debug the problem
# with inter-container communication
# https://bugs.launchpad.net/fuel/+bug/1349287
utils.exec_cmd('iptables -t nat -S')
utils.exec_cmd('iptables -S')
utils.exec_cmd('cat /etc/sysconfig/iptables')
utils.exec_cmd('cat /etc/sysconfig/iptables.save')
def stop_container(self, container_id):
"""Stop docker container
:param container_id: container id
"""
logger.debug(u'Stop container: {0}'.format(container_id))
try:
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
except requests.exceptions.Timeout:
# NOTE(eli): docker use SIGTERM signal
# to stop container if timeout expired
# docker use SIGKILL to stop container.
# Here we just want to make sure that
# container was stopped.
logger.warn(
u'Couldn\'t stop ctonainer, try '
'to stop it again: {0}'.format(container_id))
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
def start_container(self, container, **params):
"""Start containers
:param container: container name
:param params: dict of arguments for container starting
"""
logger.debug(u'Start container "{0}": {1}'.format(
container['Id'], params))
self.docker_client.start(container['Id'], **params)
def create_container(self, image_name, **params):
"""Create container
:param image_name: name of image
:param params: parameters format equals to
create_container call of docker
client
"""
# We have to delete container because we cannot
# have several containers with the same name
container_name = params.get('name')
if container_name is not None:
self._delete_container_if_exist(container_name)
new_params = deepcopy(params)
new_params['ports'] = self.get_ports(new_params)
logger.debug(u'Create container from image {0}: {1}'.format(
image_name, new_params))
def func_create():
return self.docker_client.create_container(
image_name,
**new_params)
return self.exec_with_retries(
func_create,
docker.errors.APIError,
"Can't set cookie",
retries=3,
interval=2)
def make_new_release_containers_list(self):
"""Returns list of dicts with information
for new containers.
"""
new_containers = []
for container in self.config.containers:
new_container = deepcopy(container)
new_container['image_name'] = self.make_image_name(
container['from_image'])
new_container['container_name'] = self.make_container_name(
container['id'])
new_containers.append(new_container)
return new_containers
def make_container_name(self, container_id, version=None):
"""Returns container name
:params container_id: container's id
:returns: name of the container
"""
if version is None:
version = self.config.new_version
return u'{0}{1}-{2}'.format(
self.config.container_prefix, version, container_id)
def make_new_release_images_list(self):
"""Returns list of dicts with information
for new images.
"""
new_images = []
for image in self.config.images:
new_image = deepcopy(image)
new_image['name'] = self.make_image_name(image['id'])
new_image['type'] = image['type']
new_image['docker_image'] = image['docker_image']
new_image['docker_file'] = image['docker_file']
new_images.append(new_image)
return new_images
def make_image_name(self, image_id):
"""Makes full image name
:param image_id: image id from config file
:returns: full name
"""
images = filter(
lambda i: i['id'] == image_id,
self.config.images)
if not images:
raise errors.CannotFindImageError(
'Cannot find image with id: {0}'.format(image_id))
image = images[0]
self._check_image_type(image['type'])
if image['type'] == 'base':
return image['id']
return u'{0}{1}_{2}'.format(
self.config.image_prefix,
image['id'],
self.config.new_version)
def _check_image_type(self, image_type):
"""Check if image type is valid
:param image_type: string, type of image
:raises UnsupportedImageTypeError:
"""
if not image_type in ('base', 'fuel'):
raise errors.UnsupportedImageTypeError(
'Unsupported umage type: {0}'.format(image_type))
def container_by_id(self, container_id):
"""Get container from new release by id
:param container_id: id of container
"""
filtered_containers = filter(
lambda c: c['id'] == container_id,
self.new_release_containers)
if not filtered_containers:
raise errors.CannotFindContainerError(
'Cannot find container with id {0}'.format(container_id))
return filtered_containers[0]
def container_docker_id(self, name):
"""Returns running container with specified name
:param name: name of the container
:returns: id of the container or None if not found
:raises CannotFindContainerError:
"""
containers_with_name = self._get_containers_by_name(name)
running_containers = filter(
lambda c: c['Status'].startswith('Up'),
containers_with_name)
if not running_containers:
raise errors.CannotFindContainerError(
'Cannot find running container with name "{0}"'.format(name))
return running_containers[0]['Id']
def remove_new_release_images(self):
"""We need to remove images for current release
because this script can be run several times
and we have to delete images before images
building
"""
# Don't remove base images because we cannot
# determine what version they belong to
images = filter(
lambda i: i.get('type') == 'fuel',
self.new_release_images)
image_names = [c['name'] for c in images]
for image in image_names:
self._delete_containers_for_image(image)
if self.docker_client.images(name=image):
logger.info(u'Remove image for new version {0}'.format(
image))
self.docker_client.remove_image(image)
def _delete_container_if_exist(self, container_name):
"""Deletes docker container if it exists
:param container_name: name of container
"""
found_containers = self._get_containers_by_name(container_name)
for container in found_containers:
self.stop_container(container['Id'])
logger.debug(u'Delete container {0}'.format(container))
# TODO(eli): refactor it and make retries
# as a decorator
def func_remove():
self.docker_client.remove_container(container['Id'])
self.exec_with_retries(
func_remove,
docker.errors.APIError,
'Error running removeDevice',
retries=3,
interval=2)
def _get_containers_by_name(self, container_name):
return filter(
lambda c: u'/{0}'.format(container_name) in c['Names'],
self.docker_client.containers(all=True))
def _delete_containers_for_image(self, image):
"""Deletes docker containers for specified image
:param image: name of image
"""
all_containers = self.docker_client.containers(all=True)
containers = filter(
# NOTE(eli): We must use convertation to
# str because in some cases Image is integer
lambda c: str(c.get('Image')).startswith(image),
all_containers)
for container in containers:
logger.debug(u'Try to stop container {0} which '
'depends on image {1}'.format(container['Id'], image))
self.docker_client.stop(container['Id'])
logger.debug(u'Delete container {0} which '
'depends on image {1}'.format(container['Id'], image))
self.docker_client.remove_container(container['Id'])
class DockerUpgrader(UpgradeEngine):
"""Docker management system for upgrades
"""
def __init__(self, *args, **kwargs):
super(DockerUpgrader, self).__init__(*args, **kwargs)
self.working_directory = self.config.working_directory
utils.create_dir_if_not_exists(self.working_directory)
self.docker_client = docker.Client(
base_url=self.config.docker['url'],
version=self.config.docker['api_version'],
timeout=self.config.docker['http_timeout'])
self.new_release_containers = self.make_new_release_containers_list()
self.cobbler_config_path = self.config.cobbler_config_path.format(
working_directory=self.working_directory)
self.upgrade_verifier = FuelUpgradeVerify(self.config)
self.from_version = self.config.from_version
self.supervisor = SupervisorClient(self.config, self.from_version)
self.version_file = VersionFile(self.config)
self.version_file.save_current()
def upgrade(self):
"""Method with upgarde logic
"""
# Preapre env for upgarde
self.save_db()
self.save_cobbler_configs()
# Point to new configs (version.yaml and supervisor configs) and
# restart supervisor in order to apply them
self.version_file.switch_to_new()
self.switch_to_new_configs()
self.supervisor.restart_and_wait()
# Stop docker containers (it's safe, since at this time supervisor's
# configs are empty.
self.stop_fuel_containers()
# Upload new docker images and create containers
self.upload_images()
self.create_and_start_new_containers()
# Generate supervisor configs for new containers and restart
# supervisor in order to apply them. Note, supervisor's processes
# will be attached to running docker containers automatically.
self.generate_configs(autostart=True)
self.supervisor.restart_and_wait()
# Verify that all services up and running
self.upgrade_verifier.verify()
def rollback(self):
"""Method which contains rollback logic
"""
self.version_file.switch_to_previous()
self.supervisor.switch_to_previous_configs()
self.supervisor.stop_all_services()
self.stop_fuel_containers()
self.supervisor.restart_and_wait()
def on_success(self):
"""Remove saved version files for all upgrades
NOTE(eli): It solves several problems:
1. user runs upgrade 5.0 -> 5.1 which fails
upgrade system saves version which we upgrade
from in file working_dir/5.1/version.yaml.
Then user runs upgrade 5.0 -> 5.0.1 which
successfully upgraded. Then user runs again
upgrade 5.0.1 -> 5.1, but there is saved file
working_dir/5.1/version.yaml which contains
5.0 version, and upgrade system thinks that
it's upgrading from 5.0 version, as result
it tries to make database dump from wrong
version of container.
2. without this hack user can run upgrade
second time and loose his data, this hack
prevents this case because before upgrade
checker will use current version instead
of saved version to determine version which
we run upgrade from.
"""
for version_file in glob.glob(self.config.version_files_mask):
utils.remove(version_file)
@property
def required_free_space(self):
"""Required free space to run upgrade
* space for docker
* several megabytes for configs
* reserve several megabytes for working directory
where we keep postgresql dump and cobbler configs
:returns: dict where key is path to directory
and value is required free space
"""
return {
self.config.docker['dir']: self._calculate_images_size(),
self.config.supervisor['configs_prefix']: 10,
self.config.fuel_config_path: 10,
self.working_directory: 150
}
def _calculate_images_size(self):
return utils.files_size([self.config.images])
def save_db(self):
"""Saves postgresql database into the file
"""
logger.debug(u'Backup database')
pg_dump_path = os.path.join(self.working_directory, 'pg_dump_all.sql')
pg_dump_files = utils.VersionedFile(pg_dump_path)
pg_dump_tmp_path = pg_dump_files.next_file_name()
utils.wait_for_true(
lambda: self.make_pg_dump(pg_dump_tmp_path, pg_dump_path),
timeout=self.config.db_backup_timeout,
interval=self.config.db_backup_interval)
valid_dumps = filter(utils.verify_postgres_dump,
pg_dump_files.sorted_files())
if valid_dumps:
utils.hardlink(valid_dumps[0], pg_dump_path, overwrite=True)
map(utils.remove_if_exists,
valid_dumps[self.config.keep_db_backups_count:])
else:
raise errors.DatabaseDumpError(
u'Failed to make database dump, there '
'are no valid database backup '
'files, {0}'.format(pg_dump_path))
def make_pg_dump(self, pg_dump_tmp_path, pg_dump_path):
"""Run postgresql dump in container
:param str pg_dump_tmp_path: path to temporary dump file
:param str pg_dump_path: path to dump which will be restored
in the new container, if this file is
exists, it means the user already
ran upgrade and for some reasons it
failed
:returns: True if db was successfully dumped or if dump exists
False if container isn't running or dump isn't succeed
"""
try:
container_name = self.make_container_name('postgres',
self.from_version)
self.exec_cmd_in_container(
container_name,
u"su postgres -c 'pg_dumpall --clean' > {0}".format(
pg_dump_tmp_path))
except (errors.ExecutedErrorNonZeroExitCode,
errors.CannotFindContainerError) as exc:
utils.remove_if_exists(pg_dump_tmp_path)
if not utils.file_exists(pg_dump_path):
logger.debug('Failed to make database dump %s', exc)
return False
logger.debug(
u'Failed to make database dump, '
'will be used dump from previous run: %s', exc)
return True
def save_cobbler_configs(self):
"""Copy config files from container
"""
container_name = self.make_container_name('cobbler', self.from_version)
try:
utils.exec_cmd('docker cp {0}:{1} {2}'.format(
container_name, self.config.cobbler_container_config_path,
self.cobbler_config_path))
except errors.ExecutedErrorNonZeroExitCode:
utils.rmtree(self.cobbler_config_path)
raise
self.verify_cobbler_configs()
def verify_cobbler_configs(self):
"""Verify that cobbler config directory
contains valid data
"""
configs = glob.glob(
self.config.cobbler_config_files_for_verifier.format(
cobbler_config_path=self.cobbler_config_path))
# NOTE(eli): cobbler config directory should
# contain at least one file (default.json)
if len(configs) < 1:
raise errors.WrongCobblerConfigsError(
u'Cannot find json files in directory {0}'.format(
self.cobbler_config_path))
for config in configs:
if not utils.check_file_is_valid_json(config):
raise errors.WrongCobblerConfigsError(
u'Invalid json config {0}'.format(config))
def upload_images(self):
"""Uploads images to docker
"""
logger.info(u'Start image uploading')
if not os.path.exists(self.config.images):
logger.warn(u'Cannot find docker images "%s"', self.config.images)
return
# NOTE(eli): docker-py binding
# doesn't have equal call for
# image importing which equals to
# `docker load`
utils.exec_cmd(u'docker load -i "{0}"'.format(self.config.images))
def create_and_start_new_containers(self):
"""Create containers in the right order
"""
logger.info(u'Started containers creation')
graph = self.build_dependencies_graph(self.new_release_containers)
logger.debug(u'Built dependencies graph %s', graph)
containers_to_creation = utils.topological_sorting(graph)
logger.debug(u'Resolved creation order %s', containers_to_creation)
for container_id in containers_to_creation:
container = self.container_by_id(container_id)
logger.debug(u'Start container %s', container)
links = self.get_container_links(container)
created_container = self.create_container(
container['image_name'],
name=container.get('container_name'),
volumes=container.get('volumes'),
ports=container.get('ports'),
detach=False)
volumes_from = []
for container_id in container.get('volumes_from', []):
volume_container = self.container_by_id(container_id)
volumes_from.append(volume_container['container_name'])
# NOTE(ikalnitsky):
# Conflicting options: --net=host can't be used with links.
# Still, we need links at least for resolving containers
# start order.
if container.get('network_mode') == 'host':
links = None
self.start_container(created_container,
port_bindings=container.get('port_bindings'),
links=links,
volumes_from=volumes_from,
binds=container.get('binds'),
network_mode=container.get('network_mode'),
privileged=container.get('privileged', False))
if container.get('after_container_creation_command'):
self.run_after_container_creation_command(container)
def run_after_container_creation_command(self, container):
"""Runs command in container with retries in
case of error
:param container: dict with container information
"""
command = container['after_container_creation_command']
def execute():
self.exec_cmd_in_container(container['container_name'], command)
self.exec_with_retries(execute,
errors.ExecutedErrorNonZeroExitCode,
'',
retries=30,
interval=4)
def exec_cmd_in_container(self, container_name, cmd):
"""Execute command in running container
:param name: name of the container, like fuel-core-5.1-nailgun
"""
db_container_id = self.container_docker_id(container_name)
utils.exec_cmd("dockerctl shell {0} {1}".format(db_container_id, cmd))
def get_ports(self, container):
"""Docker binding accepts ports as tuple,
here we convert from list to tuple.
FIXME(eli): https://github.com/dotcloud/docker-py/blob/
73434476b32136b136e1cdb0913fd123126f2a52/
docker/client.py#L111-L114
"""
ports = container.get('ports')
if ports is None:
return
return [
port if not isinstance(port, list) else tuple(port)
for port in ports
]
def exec_with_retries(self,
func,
exceptions,
message,
retries=0,
interval=0):
# TODO(eli): refactor it and make retries
# as a decorator
intervals = retries * [interval]
for interval in intervals:
try:
return func()
except exceptions as exc:
if str(exc).endswith(message):
time.sleep(interval)
continue
raise
return func()
def get_container_links(self, container):
links = []
if container.get('links'):
for container_link in container.get('links'):
link_container = self.container_by_id(container_link['id'])
links.append((link_container['container_name'],
container_link['alias']))
return links
@classmethod
def build_dependencies_graph(cls, containers):
"""Builds graph which based on
`volumes_from` and `link` parameters
of container.
:returns: dict where keys are nodes and
values are lists of dependencies
"""
graph = {}
for container in containers:
graph[container['id']] = sorted(
set(
container.get('volumes_from', []) +
[link['id'] for link in container.get('links', [])]))
return graph
def generate_configs(self, autostart=True):
"""Generates supervisor configs
and saves them to configs directory
"""
configs = []
for container in self.new_release_containers:
params = {
'config_name':
container['id'],
'service_name':
self.make_service_name(container['id']),
'command':
u'docker start -a {0}'.format(container['container_name']),
'autostart':
autostart
}
if container['supervisor_config']:
configs.append(params)
self.supervisor.generate_configs(configs)
def make_service_name(self, container_name):
return 'docker-{0}'.format(container_name)
def switch_to_new_configs(self):
"""Switches supervisor to new configs
"""
self.supervisor.switch_to_new_configs()
def volumes_dependencies(self, container):
"""Get list of `volumes` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'volumes_from')
def link_dependencies(self, container):
"""Get list of `link` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'link')
def dependencies_names(self, container, key):
"""Returns list of dependencies for specified key
:param contaienr: dict with information about container
:param key: key which will be used for dependencies retrieving
:returns: list of container names
"""
names = []
if container.get(key):
for container_id in container.get(key):
container = self.container_by_id(container_id)
names.append(container['container_name'])
return names
def stop_fuel_containers(self):
"""Use docker API to shutdown containers
"""
containers = self.docker_client.containers(limit=-1)
containers_to_stop = filter(
lambda c: c['Image'].startswith(self.config.image_prefix),
containers)
for container in containers_to_stop:
logger.debug(u'Stop container: %s', container)
self.stop_container(container['Id'])
def _get_docker_container_public_ports(self, containers):
"""Returns public ports
:param containers: list of dicts with information about
containers which have `Ports` list
with items where exist `PublicPort`
field
:returns: list of public ports
"""
container_ports = []
for container in containers:
container_ports.extend(container['Ports'])
return [
container_port['PublicPort'] for container_port in container_ports
]
def stop_container(self, container_id):
"""Stop docker container
:param container_id: container id
"""
logger.debug(u'Stop container: %s', container_id)
try:
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
except requests.exceptions.Timeout:
# NOTE(eli): docker use SIGTERM signal
# to stop container if timeout expired
# docker use SIGKILL to stop container.
# Here we just want to make sure that
# container was stopped.
logger.warn(
u'Couldn\'t stop ctonainer, try '
'to stop it again: %s', container_id)
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
def start_container(self, container, **params):
"""Start containers
:param container: container name
:param params: dict of arguments for container starting
"""
logger.debug(u'Start container "%s": %s', container['Id'], params)
self.docker_client.start(container['Id'], **params)
def create_container(self, image_name, **params):
"""Create container
:param image_name: name of image
:param params: parameters format equals to
create_container call of docker
client
"""
# We have to delete container because we cannot
# have several containers with the same name
container_name = params.get('name')
if container_name is not None:
self._delete_container_if_exist(container_name)
new_params = deepcopy(params)
new_params['ports'] = self.get_ports(new_params)
logger.debug(u'Create container from image %s: %s', image_name,
new_params)
def func_create():
return self.docker_client.create_container(image_name,
**new_params)
return self.exec_with_retries(func_create,
docker.errors.APIError,
"Can't set cookie",
retries=3,
interval=2)
def make_new_release_containers_list(self):
"""Returns list of dicts with information
for new containers.
"""
new_containers = []
for container in self.config.containers:
new_container = deepcopy(container)
new_container['image_name'] = self.make_image_name(
container['from_image'])
new_container['container_name'] = self.make_container_name(
container['id'])
new_containers.append(new_container)
return new_containers
def make_container_name(self, container_id, version=None):
"""Returns container name
:params container_id: container's id
:returns: name of the container
"""
if version is None:
version = self.config.new_version
return u'{0}{1}-{2}'.format(self.config.container_prefix, version,
container_id)
def make_image_name(self, image_id):
"""Makes full image name
:param image_id: image id from config file
:returns: full name
"""
return u'{0}{1}_{2}'.format(self.config.image_prefix, image_id,
self.config.new_version)
def container_by_id(self, container_id):
"""Get container from new release by id
:param container_id: id of container
"""
filtered_containers = filter(lambda c: c['id'] == container_id,
self.new_release_containers)
if not filtered_containers:
raise errors.CannotFindContainerError(
'Cannot find container with id {0}'.format(container_id))
return filtered_containers[0]
def container_docker_id(self, name):
"""Returns running container with specified name
:param name: name of the container
:returns: id of the container or None if not found
:raises CannotFindContainerError:
"""
containers_with_name = self._get_containers_by_name(name)
running_containers = filter(lambda c: c['Status'].startswith('Up'),
containers_with_name)
if not running_containers:
raise errors.CannotFindContainerError(
'Cannot find running container with name "{0}"'.format(name))
return running_containers[0]['Id']
def _delete_container_if_exist(self, container_name):
"""Deletes docker container if it exists
:param container_name: name of container
"""
found_containers = self._get_containers_by_name(container_name)
for container in found_containers:
self.stop_container(container['Id'])
logger.debug(u'Delete container %s', container)
# TODO(eli): refactor it and make retries
# as a decorator
def func_remove():
self.docker_client.remove_container(container['Id'])
self.exec_with_retries(func_remove,
docker.errors.APIError,
'Error running removeDevice',
retries=3,
interval=2)
def _get_containers_by_name(self, container_name):
return filter(lambda c: u'/{0}'.format(container_name) in c['Names'],
self.docker_client.containers(all=True))
def _delete_containers_for_image(self, image):
"""Deletes docker containers for specified image
:param image: name of image
"""
all_containers = self.docker_client.containers(all=True)
containers = filter(
# NOTE(eli): We must use convertation to
# str because in some cases Image is integer
lambda c: str(c.get('Image')).startswith(image),
all_containers)
for container in containers:
logger.debug(
u'Try to stop container %s which '
'depends on image %s', container['Id'], image)
self.docker_client.stop(container['Id'])
logger.debug(u'Delete container %s which '
'depends on image %s', container['Id'], image)
self.docker_client.remove_container(container['Id'])
class DockerUpgrader(UpgradeEngine):
"""Docker management system for upgrades
"""
def __init__(self, *args, **kwargs):
super(DockerUpgrader, self).__init__(*args, **kwargs)
self.working_directory = self.config.working_directory
utils.create_dir_if_not_exists(self.working_directory)
self.docker_client = docker.Client(
base_url=self.config.docker['url'],
version=self.config.docker['api_version'],
timeout=self.config.docker['http_timeout'])
self.new_release_images = self.make_new_release_images_list()
self.new_release_containers = self.make_new_release_containers_list()
self.cobbler_config_path = self.config.cobbler_config_path.format(
working_directory=self.working_directory)
self.upgrade_verifier = FuelUpgradeVerify(self.config)
self.from_version = self.config.from_version
self.supervisor = SupervisorClient(self.config, self.from_version)
self.version_file = VersionFile(self.config)
self.version_file.save_current()
def upgrade(self):
"""Method with upgarde logic
"""
# Preapre env for upgarde
self.save_db()
self.save_cobbler_configs()
self.save_astute_keys()
# NOTE(akislitsky): fix for bug
# https://bugs.launchpad.net/fuel/+bug/1354465
# supervisord can restart old container even if it already stopped
# and new container start will be failed. We switch configs
# before upgrade, so if supervisord will try to start container
# it will be new container.
self.switch_to_new_configs()
# Stop all of the services
self.supervisor.stop_all_services()
self.stop_fuel_containers()
# Upload docker images
self.upload_images()
# Generate config with autostart is false
# not to run new services on supervisor restart
self.generate_configs(autostart=False)
self.version_file.switch_to_new()
# Restart supervisor to read new configs
self.supervisor.restart_and_wait()
# Create container and start it under supervisor
self.create_and_start_new_containers()
# Update configs in order to start all
# services on supervisor restart
self.generate_configs(autostart=True)
# Verify that all services up and running
self.upgrade_verifier.verify()
def rollback(self):
"""Method which contains rollback logic
"""
self.version_file.switch_to_previous()
self.supervisor.switch_to_previous_configs()
self.supervisor.stop_all_services()
self.stop_fuel_containers()
self.supervisor.restart_and_wait()
def on_success(self):
"""Remove saved version files for all upgrades
NOTE(eli): It solves several problems:
1. user runs upgrade 5.0 -> 5.1 which fails
upgrade system saves version which we upgrade
from in file working_dir/5.1/version.yaml.
Then user runs upgrade 5.0 -> 5.0.1 which
successfully upgraded. Then user runs again
upgrade 5.0.1 -> 5.1, but there is saved file
working_dir/5.1/version.yaml which contains
5.0 version, and upgrade system thinks that
it's upgrading from 5.0 version, as result
it tries to make database dump from wrong
version of container.
2. without this hack user can run upgrade
second time and loose his data, this hack
prevents this case because before upgrade
checker will use current version instead
of saved version to determine version which
we run upgrade from.
"""
for version_file in glob.glob(self.config.version_files_mask):
utils.remove(version_file)
@property
def required_free_space(self):
"""Required free space to run upgrade
* space for docker
* several megabytes for configs
* reserve several megabytes for working directory
where we keep postgresql dump and cobbler configs
:returns: dict where key is path to directory
and value is required free space
"""
return {
self.config.docker['dir']: self._calculate_images_size(),
self.config.supervisor['configs_prefix']: 10,
self.config.fuel_config_path: 10,
self.working_directory: 150}
def _calculate_images_size(self):
images_list = [i['docker_image'] for i in self.new_release_images]
return utils.files_size(images_list)
def save_db(self):
"""Saves postgresql database into the file
"""
logger.debug(u'Backup database')
pg_dump_path = os.path.join(self.working_directory, 'pg_dump_all.sql')
pg_dump_files = utils.VersionedFile(pg_dump_path)
pg_dump_tmp_path = pg_dump_files.next_file_name()
utils.wait_for_true(
lambda: self.make_pg_dump(pg_dump_tmp_path, pg_dump_path),
timeout=self.config.db_backup_timeout,
interval=self.config.db_backup_interval)
valid_dumps = filter(utils.verify_postgres_dump,
pg_dump_files.sorted_files())
if valid_dumps:
utils.hardlink(valid_dumps[0], pg_dump_path, overwrite=True)
map(utils.remove_if_exists,
valid_dumps[self.config.keep_db_backups_count:])
else:
raise errors.DatabaseDumpError(
u'Failed to make database dump, there '
'are no valid database backup '
'files, {0}'.format(pg_dump_path))
def make_pg_dump(self, pg_dump_tmp_path, pg_dump_path):
"""Run postgresql dump in container
:param str pg_dump_tmp_path: path to temporary dump file
:param str pg_dump_path: path to dump which will be restored
in the new container, if this file is
exists, it means the user already
ran upgrade and for some reasons it
failed
:returns: True if db was successfully dumped or if dump exists
False if container isn't running or dump isn't succeed
"""
try:
container_name = self.make_container_name(
'postgres', self.from_version)
self.exec_cmd_in_container(
container_name,
u"su postgres -c 'pg_dumpall --clean' > {0}".format(
pg_dump_tmp_path))
except (errors.ExecutedErrorNonZeroExitCode,
errors.CannotFindContainerError) as exc:
utils.remove_if_exists(pg_dump_tmp_path)
if not utils.file_exists(pg_dump_path):
logger.debug('Failed to make database dump {0}'.format(exc))
return False
logger.debug(
u'Failed to make database dump, '
'will be used dump from previous run: %s', exc)
return True
def save_astute_keys(self):
"""Copy any astute generated keys."""
container_name = self.make_container_name('astute', self.from_version)
utils.remove(self.config.astute_keys_path)
try:
utils.exec_cmd('docker cp {0}:{1} {2}'.format(
container_name,
self.config.astute_container_keys_path,
self.config.working_directory))
except errors.ExecutedErrorNonZeroExitCode as exc:
# If there was error, mostly it's because of error
#
# Error: Could not find the file /var/lib/astute
# in container fuel-core-5.0-astute
#
# It means that user didn't run deployment on his
# env, because this directory is created by orchestrator
# during the first deployment.
# Also it can fail if there was no running container
# in both case we should create empty directory to copy
# it in after container creation section
logger.debug(
'Cannot copy astute keys, creating empty directory '
'%s: %s', self.config.astute_keys_path, exc)
if not utils.file_exists(self.config.astute_keys_path):
os.mkdir(self.config.astute_keys_path)
def save_cobbler_configs(self):
"""Copy config files from container
"""
container_name = self.make_container_name(
'cobbler', self.from_version)
try:
utils.exec_cmd('docker cp {0}:{1} {2}'.format(
container_name,
self.config.cobbler_container_config_path,
self.cobbler_config_path))
except errors.ExecutedErrorNonZeroExitCode:
utils.rmtree(self.cobbler_config_path)
raise
self.verify_cobbler_configs()
def verify_cobbler_configs(self):
"""Verify that cobbler config directory
contains valid data
"""
configs = glob.glob(
self.config.cobbler_config_files_for_verifier.format(
cobbler_config_path=self.cobbler_config_path))
# NOTE(eli): cobbler config directory should
# contain at least one file (default.json)
if len(configs) < 1:
raise errors.WrongCobblerConfigsError(
u'Cannot find json files in directory {0}'.format(
self.cobbler_config_path))
for config in configs:
if not utils.check_file_is_valid_json(config):
raise errors.WrongCobblerConfigsError(
u'Invalid json config {0}'.format(config))
def upload_images(self):
"""Uploads images to docker
"""
logger.info(u'Start image uploading')
for image in self.new_release_images:
logger.debug(u'Try to upload docker image {0}'.format(image))
docker_image = image['docker_image']
if not os.path.exists(docker_image):
logger.warn(u'Cannot find docker image "{0}"'.format(
docker_image))
continue
# NOTE(eli): docker-py binding
# doesn't have equal call for
# image importing which equals to
# `docker load`
utils.exec_cmd(u'docker load < "{0}"'.format(docker_image))
def create_and_start_new_containers(self):
"""Create containers in the right order
"""
logger.info(u'Started containers creation')
graph = self.build_dependencies_graph(self.new_release_containers)
logger.debug(u'Built dependencies graph {0}'.format(graph))
containers_to_creation = utils.topological_sorting(graph)
logger.debug(u'Resolved creation order {0}'.format(
containers_to_creation))
for container_id in containers_to_creation:
container = self.container_by_id(container_id)
logger.debug(u'Start container {0}'.format(container))
links = self.get_container_links(container)
created_container = self.create_container(
container['image_name'],
name=container.get('container_name'),
volumes=container.get('volumes'),
ports=container.get('ports'),
detach=False)
volumes_from = []
for container_id in container.get('volumes_from', []):
volume_container = self.container_by_id(container_id)
volumes_from.append(volume_container['container_name'])
self.start_container(
created_container,
port_bindings=container.get('port_bindings'),
links=links,
volumes_from=volumes_from,
binds=container.get('binds'),
privileged=container.get('privileged', False))
if container.get('after_container_creation_command'):
self.run_after_container_creation_command(container)
if container.get('supervisor_config'):
self.start_service_under_supervisor(
self.make_service_name(container['id']))
self.clean_iptables_rules(container)
def run_after_container_creation_command(self, container):
"""Runs command in container with retries in
case of error
:param container: dict with container information
"""
command = container['after_container_creation_command']
def execute():
self.exec_cmd_in_container(container['container_name'], command)
self.exec_with_retries(
execute, errors.ExecutedErrorNonZeroExitCode,
'', retries=30, interval=4)
def exec_cmd_in_container(self, container_name, cmd):
"""Execute command in running container
:param name: name of the container, like fuel-core-5.1-nailgun
"""
db_container_id = self.container_docker_id(container_name)
# NOTE(eli): we don't use dockerctl shell
# instead of lxc-attach here because
# release 5.0 has a bug which doesn't
# allow us to use quotes in command
# https://bugs.launchpad.net/fuel/+bug/1324200
utils.exec_cmd(
"lxc-attach --name {0} -- {1}".format(
db_container_id, cmd))
def get_ports(self, container):
"""Docker binding accepts ports as tuple,
here we convert from list to tuple.
FIXME(eli): https://github.com/dotcloud/docker-py/blob/
73434476b32136b136e1cdb0913fd123126f2a52/
docker/client.py#L111-L114
"""
ports = container.get('ports')
if ports is None:
return
return [port if not isinstance(port, list) else tuple(port)
for port in ports]
def start_service_under_supervisor(self, service_name):
"""Start service under supervisor
:param str service_name: name of the service
"""
self.supervisor.start(service_name)
def exec_with_retries(
self, func, exceptions, message, retries=0, interval=0):
# TODO(eli): refactor it and make retries
# as a decorator
intervals = retries * [interval]
for interval in intervals:
try:
return func()
except exceptions as exc:
if str(exc).endswith(message):
time.sleep(interval)
continue
raise
return func()
def get_container_links(self, container):
links = []
if container.get('links'):
for container_link in container.get('links'):
link_container = self.container_by_id(
container_link['id'])
links.append((
link_container['container_name'],
container_link['alias']))
return links
@classmethod
def build_dependencies_graph(cls, containers):
"""Builds graph which based on
`volumes_from` and `link` parameters
of container.
:returns: dict where keys are nodes and
values are lists of dependencies
"""
graph = {}
for container in containers:
graph[container['id']] = sorted(set(
container.get('volumes_from', []) +
[link['id'] for link in container.get('links', [])]))
return graph
def generate_configs(self, autostart=True):
"""Generates supervisor configs
and saves them to configs directory
"""
configs = []
for container in self.new_release_containers:
params = {
'config_name': container['id'],
'service_name': self.make_service_name(container['id']),
'command': u'docker start -a {0}'.format(
container['container_name']),
'autostart': autostart
}
if container['supervisor_config']:
configs.append(params)
self.supervisor.generate_configs(configs)
cobbler_container = self.container_by_id('cobbler')
self.supervisor.generate_cobbler_config(
cobbler_container['id'],
self.make_service_name(cobbler_container['id']),
cobbler_container['container_name'],
autostart=autostart)
def make_service_name(self, container_name):
return 'docker-{0}'.format(container_name)
def switch_to_new_configs(self):
"""Switches supervisor to new configs
"""
self.supervisor.switch_to_new_configs()
def build_images(self):
"""Use docker API to build new containers
"""
self.remove_new_release_images()
for image in self.new_release_images:
logger.info(u'Start image building: {0}'.format(image))
self.docker_client.build(
path=image['docker_file'],
tag=image['name'],
nocache=True)
# NOTE(eli): 0.10 and early versions of
# Docker api dont't return correct http
# response in case of failed build, here
# we check if build succed and raise error
# if it failed i.e. image was not created
if not self.docker_client.images(name=image):
raise errors.DockerFailedToBuildImageError(
u'Failed to build image {0}'.format(image))
def volumes_dependencies(self, container):
"""Get list of `volumes` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'volumes_from')
def link_dependencies(self, container):
"""Get list of `link` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'link')
def dependencies_names(self, container, key):
"""Returns list of dependencies for specified key
:param contaienr: dict with information about container
:param key: key which will be used for dependencies retrieving
:returns: list of container names
"""
names = []
if container.get(key):
for container_id in container.get(key):
container = self.container_by_id(container_id)
names.append(container['container_name'])
return names
def stop_fuel_containers(self):
"""Use docker API to shutdown containers
"""
containers = self.docker_client.containers(limit=-1)
containers_to_stop = filter(
lambda c: c['Image'].startswith(self.config.image_prefix),
containers)
for container in containers_to_stop:
logger.debug(u'Stop container: {0}'.format(container))
self.stop_container(container['Id'])
def _get_docker_container_public_ports(self, containers):
"""Returns public ports
:param containers: list of dicts with information about
containers which have `Ports` list
with items where exist `PublicPort`
field
:returns: list of public ports
"""
container_ports = []
for container in containers:
container_ports.extend(container['Ports'])
return [container_port['PublicPort']
for container_port in container_ports]
def clean_iptables_rules(self, container):
"""Sometimes when we run docker stop
(version dc9c28f/0.10.0) it doesn't clean
iptables rules, as result when we run new
container on the same port we have two rules
with the same port but with different IPs,
we have to clean this rules to prevent services
unavailability.
Example of the problem:
$ iptables -t nat -S
...
-A DOCKER -p tcp -m tcp --dport 443 -j DNAT \
--to-destination 172.17.0.7:443
-A DOCKER -p tcp -m tcp --dport 443 -j DNAT \
--to-destination 172.17.0.3:443
-A DOCKER -d 10.108.0.2/32 -p tcp -m tcp --dport \
8777 -j DNAT --to-destination 172.17.0.10:8777
-A DOCKER -d 127.0.0.1/32 -p tcp -m tcp --dport \
8777 -j DNAT --to-destination 172.17.0.11:8777
-A DOCKER -d 10.108.0.2/32 -p tcp -m tcp --dport \
8777 -j DNAT --to-destination 172.17.0.11:8777
"""
if not container.get('port_bindings'):
return
self._log_iptables()
utils.safe_exec_cmd('dockerctl post_start_hooks {0}'.format(
container['id']))
utils.safe_exec_cmd('service iptables save')
self._log_iptables()
def _log_iptables(self):
"""Method for additional logging of iptables rules
NOTE(eli): Sometimes there are problems with
iptables rules like this
https://bugs.launchpad.net/fuel/+bug/1349287
"""
utils.safe_exec_cmd('iptables -t nat -S')
utils.safe_exec_cmd('iptables -S')
utils.safe_exec_cmd('cat /etc/sysconfig/iptables.save')
def stop_container(self, container_id):
"""Stop docker container
:param container_id: container id
"""
logger.debug(u'Stop container: {0}'.format(container_id))
try:
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
except requests.exceptions.Timeout:
# NOTE(eli): docker use SIGTERM signal
# to stop container if timeout expired
# docker use SIGKILL to stop container.
# Here we just want to make sure that
# container was stopped.
logger.warn(
u'Couldn\'t stop ctonainer, try '
'to stop it again: {0}'.format(container_id))
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
def start_container(self, container, **params):
"""Start containers
:param container: container name
:param params: dict of arguments for container starting
"""
logger.debug(u'Start container "{0}": {1}'.format(
container['Id'], params))
self.docker_client.start(container['Id'], **params)
def create_container(self, image_name, **params):
"""Create container
:param image_name: name of image
:param params: parameters format equals to
create_container call of docker
client
"""
# We have to delete container because we cannot
# have several containers with the same name
container_name = params.get('name')
if container_name is not None:
self._delete_container_if_exist(container_name)
new_params = deepcopy(params)
new_params['ports'] = self.get_ports(new_params)
logger.debug(u'Create container from image {0}: {1}'.format(
image_name, new_params))
def func_create():
return self.docker_client.create_container(
image_name,
**new_params)
return self.exec_with_retries(
func_create,
docker.errors.APIError,
"Can't set cookie",
retries=3,
interval=2)
def make_new_release_containers_list(self):
"""Returns list of dicts with information
for new containers.
"""
new_containers = []
for container in self.config.containers:
new_container = deepcopy(container)
new_container['image_name'] = self.make_image_name(
container['from_image'])
new_container['container_name'] = self.make_container_name(
container['id'])
new_containers.append(new_container)
return new_containers
def make_container_name(self, container_id, version=None):
"""Returns container name
:params container_id: container's id
:returns: name of the container
"""
if version is None:
version = self.config.new_version
return u'{0}{1}-{2}'.format(
self.config.container_prefix, version, container_id)
def make_new_release_images_list(self):
"""Returns list of dicts with information
for new images.
"""
new_images = []
for image in self.config.images:
new_image = deepcopy(image)
new_image['name'] = self.make_image_name(image['id'])
new_image['type'] = image['type']
new_image['docker_image'] = image['docker_image']
new_image['docker_file'] = image['docker_file']
new_images.append(new_image)
return new_images
def make_image_name(self, image_id):
"""Makes full image name
:param image_id: image id from config file
:returns: full name
"""
images = filter(
lambda i: i['id'] == image_id,
self.config.images)
if not images:
raise errors.CannotFindImageError(
'Cannot find image with id: {0}'.format(image_id))
image = images[0]
self._check_image_type(image['type'])
if image['type'] == 'base':
return image['id']
return u'{0}{1}_{2}'.format(
self.config.image_prefix,
image['id'],
self.config.new_version)
def _check_image_type(self, image_type):
"""Check if image type is valid
:param image_type: string, type of image
:raises UnsupportedImageTypeError:
"""
if not image_type in ('base', 'fuel'):
raise errors.UnsupportedImageTypeError(
'Unsupported umage type: {0}'.format(image_type))
def container_by_id(self, container_id):
"""Get container from new release by id
:param container_id: id of container
"""
filtered_containers = filter(
lambda c: c['id'] == container_id,
self.new_release_containers)
if not filtered_containers:
raise errors.CannotFindContainerError(
'Cannot find container with id {0}'.format(container_id))
return filtered_containers[0]
def container_docker_id(self, name):
"""Returns running container with specified name
:param name: name of the container
:returns: id of the container or None if not found
:raises CannotFindContainerError:
"""
containers_with_name = self._get_containers_by_name(name)
running_containers = filter(
lambda c: c['Status'].startswith('Up'),
containers_with_name)
if not running_containers:
raise errors.CannotFindContainerError(
'Cannot find running container with name "{0}"'.format(name))
return running_containers[0]['Id']
def remove_new_release_images(self):
"""We need to remove images for current release
because this script can be run several times
and we have to delete images before images
building
"""
# Don't remove base images because we cannot
# determine what version they belong to
images = filter(
lambda i: i.get('type') == 'fuel',
self.new_release_images)
image_names = [c['name'] for c in images]
for image in image_names:
self._delete_containers_for_image(image)
if self.docker_client.images(name=image):
logger.info(u'Remove image for new version {0}'.format(
image))
self.docker_client.remove_image(image)
def _delete_container_if_exist(self, container_name):
"""Deletes docker container if it exists
:param container_name: name of container
"""
found_containers = self._get_containers_by_name(container_name)
for container in found_containers:
self.stop_container(container['Id'])
logger.debug(u'Delete container {0}'.format(container))
# TODO(eli): refactor it and make retries
# as a decorator
def func_remove():
self.docker_client.remove_container(container['Id'])
self.exec_with_retries(
func_remove,
docker.errors.APIError,
'Error running removeDevice',
retries=3,
interval=2)
def _get_containers_by_name(self, container_name):
return filter(
lambda c: u'/{0}'.format(container_name) in c['Names'],
self.docker_client.containers(all=True))
def _delete_containers_for_image(self, image):
"""Deletes docker containers for specified image
:param image: name of image
"""
all_containers = self.docker_client.containers(all=True)
containers = filter(
# NOTE(eli): We must use convertation to
# str because in some cases Image is integer
lambda c: str(c.get('Image')).startswith(image),
all_containers)
for container in containers:
logger.debug(u'Try to stop container {0} which '
'depends on image {1}'.format(container['Id'], image))
self.docker_client.stop(container['Id'])
logger.debug(u'Delete container {0} which '
'depends on image {1}'.format(container['Id'], image))
self.docker_client.remove_container(container['Id'])
class DockerUpgrader(UpgradeEngine):
"""Docker management system for upgrades
"""
def __init__(self, *args, **kwargs):
super(DockerUpgrader, self).__init__(*args, **kwargs)
self.working_directory = self.config.working_directory
utils.create_dir_if_not_exists(self.working_directory)
self.docker_client = docker.Client(
base_url=self.config.docker['url'],
version=self.config.docker['api_version'],
timeout=self.config.docker['http_timeout'])
self.new_release_containers = self.make_new_release_containers_list()
self.cobbler_config_path = self.config.cobbler_config_path.format(
working_directory=self.working_directory)
self.upgrade_verifier = FuelUpgradeVerify(self.config)
self.from_version = self.config.from_version
self.supervisor = SupervisorClient(self.config, self.from_version)
self.version_file = VersionFile(self.config)
self.version_file.save_current()
def upgrade(self):
"""Method with upgarde logic
"""
# Preapre env for upgarde
self.save_db()
self.save_cobbler_configs()
# Point to new configs (version.yaml and supervisor configs) and
# restart supervisor in order to apply them
self.version_file.switch_to_new()
self.switch_to_new_configs()
self.supervisor.restart_and_wait()
# Stop docker containers (it's safe, since at this time supervisor's
# configs are empty.
self.stop_fuel_containers()
# Upload new docker images and create containers
self.upload_images()
self.create_and_start_new_containers()
# Generate supervisor configs for new containers and restart
# supervisor in order to apply them. Note, supervisor's processes
# will be attached to running docker containers automatically.
self.generate_configs(autostart=True)
self.supervisor.restart_and_wait()
# Verify that all services up and running
self.upgrade_verifier.verify()
def rollback(self):
"""Method which contains rollback logic
"""
self.version_file.switch_to_previous()
self.supervisor.switch_to_previous_configs()
self.supervisor.stop_all_services()
self.stop_fuel_containers()
self.supervisor.restart_and_wait()
def on_success(self):
"""Remove saved version files for all upgrades
NOTE(eli): It solves several problems:
1. user runs upgrade 5.0 -> 5.1 which fails
upgrade system saves version which we upgrade
from in file working_dir/5.1/version.yaml.
Then user runs upgrade 5.0 -> 5.0.1 which
successfully upgraded. Then user runs again
upgrade 5.0.1 -> 5.1, but there is saved file
working_dir/5.1/version.yaml which contains
5.0 version, and upgrade system thinks that
it's upgrading from 5.0 version, as result
it tries to make database dump from wrong
version of container.
2. without this hack user can run upgrade
second time and loose his data, this hack
prevents this case because before upgrade
checker will use current version instead
of saved version to determine version which
we run upgrade from.
"""
for version_file in glob.glob(self.config.version_files_mask):
utils.remove(version_file)
@property
def required_free_space(self):
"""Required free space to run upgrade
* space for docker
* several megabytes for configs
* reserve several megabytes for working directory
where we keep postgresql dump and cobbler configs
:returns: dict where key is path to directory
and value is required free space
"""
return {
self.config.docker['dir']: self._calculate_images_size(),
self.config.supervisor['configs_prefix']: 10,
self.config.fuel_config_path: 10,
self.working_directory: 150}
def _calculate_images_size(self):
return utils.files_size([self.config.images])
def save_db(self):
"""Saves postgresql database into the file
"""
logger.debug(u'Backup database')
pg_dump_path = os.path.join(self.working_directory, 'pg_dump_all.sql')
pg_dump_files = utils.VersionedFile(pg_dump_path)
pg_dump_tmp_path = pg_dump_files.next_file_name()
utils.wait_for_true(
lambda: self.make_pg_dump(pg_dump_tmp_path, pg_dump_path),
timeout=self.config.db_backup_timeout,
interval=self.config.db_backup_interval)
valid_dumps = filter(utils.verify_postgres_dump,
pg_dump_files.sorted_files())
if valid_dumps:
utils.hardlink(valid_dumps[0], pg_dump_path, overwrite=True)
map(utils.remove_if_exists,
valid_dumps[self.config.keep_db_backups_count:])
else:
raise errors.DatabaseDumpError(
u'Failed to make database dump, there '
'are no valid database backup '
'files, {0}'.format(pg_dump_path))
def make_pg_dump(self, pg_dump_tmp_path, pg_dump_path):
"""Run postgresql dump in container
:param str pg_dump_tmp_path: path to temporary dump file
:param str pg_dump_path: path to dump which will be restored
in the new container, if this file is
exists, it means the user already
ran upgrade and for some reasons it
failed
:returns: True if db was successfully dumped or if dump exists
False if container isn't running or dump isn't succeed
"""
try:
container_name = self.make_container_name(
'postgres', self.from_version)
self.exec_cmd_in_container(
container_name,
u"su postgres -c 'pg_dumpall --clean' > {0}".format(
pg_dump_tmp_path))
except (errors.ExecutedErrorNonZeroExitCode,
errors.CannotFindContainerError) as exc:
utils.remove_if_exists(pg_dump_tmp_path)
if not utils.file_exists(pg_dump_path):
logger.debug('Failed to make database dump %s', exc)
return False
logger.debug(
u'Failed to make database dump, '
'will be used dump from previous run: %s', exc)
return True
def save_cobbler_configs(self):
"""Copy config files from container
"""
container_name = self.make_container_name(
'cobbler', self.from_version)
try:
utils.exec_cmd('docker cp {0}:{1} {2}'.format(
container_name,
self.config.cobbler_container_config_path,
self.cobbler_config_path))
except errors.ExecutedErrorNonZeroExitCode:
utils.rmtree(self.cobbler_config_path)
raise
self.verify_cobbler_configs()
def verify_cobbler_configs(self):
"""Verify that cobbler config directory
contains valid data
"""
configs = glob.glob(
self.config.cobbler_config_files_for_verifier.format(
cobbler_config_path=self.cobbler_config_path))
# NOTE(eli): cobbler config directory should
# contain at least one file (default.json)
if len(configs) < 1:
raise errors.WrongCobblerConfigsError(
u'Cannot find json files in directory {0}'.format(
self.cobbler_config_path))
for config in configs:
if not utils.check_file_is_valid_json(config):
raise errors.WrongCobblerConfigsError(
u'Invalid json config {0}'.format(config))
def upload_images(self):
"""Uploads images to docker
"""
logger.info(u'Start image uploading')
if not os.path.exists(self.config.images):
logger.warn(u'Cannot find docker images "%s"', self.config.images)
return
# NOTE(eli): docker-py binding
# doesn't have equal call for
# image importing which equals to
# `docker load`
utils.exec_cmd(u'docker load -i "{0}"'.format(self.config.images))
def create_and_start_new_containers(self):
"""Create containers in the right order
"""
logger.info(u'Started containers creation')
graph = self.build_dependencies_graph(self.new_release_containers)
logger.debug(u'Built dependencies graph %s', graph)
containers_to_creation = utils.topological_sorting(graph)
logger.debug(u'Resolved creation order %s', containers_to_creation)
for container_id in containers_to_creation:
container = self.container_by_id(container_id)
logger.debug(u'Start container %s', container)
links = self.get_container_links(container)
created_container = self.create_container(
container['image_name'],
name=container.get('container_name'),
volumes=container.get('volumes'),
ports=container.get('ports'),
detach=False)
volumes_from = []
for container_id in container.get('volumes_from', []):
volume_container = self.container_by_id(container_id)
volumes_from.append(volume_container['container_name'])
# NOTE(ikalnitsky):
# Conflicting options: --net=host can't be used with links.
# Still, we need links at least for resolving containers
# start order.
if container.get('network_mode') == 'host':
links = None
self.start_container(
created_container,
port_bindings=container.get('port_bindings'),
links=links,
volumes_from=volumes_from,
binds=container.get('binds'),
network_mode=container.get('network_mode'),
privileged=container.get('privileged', False))
if container.get('after_container_creation_command'):
self.run_after_container_creation_command(container)
def run_after_container_creation_command(self, container):
"""Runs command in container with retries in
case of error
:param container: dict with container information
"""
command = container['after_container_creation_command']
def execute():
self.exec_cmd_in_container(container['container_name'], command)
self.exec_with_retries(
execute, errors.ExecutedErrorNonZeroExitCode,
'', retries=30, interval=4)
def exec_cmd_in_container(self, container_name, cmd):
"""Execute command in running container
:param name: name of the container, like fuel-core-5.1-nailgun
"""
db_container_id = self.container_docker_id(container_name)
utils.exec_cmd("dockerctl shell {0} {1}".format(db_container_id, cmd))
def get_ports(self, container):
"""Docker binding accepts ports as tuple,
here we convert from list to tuple.
FIXME(eli): https://github.com/dotcloud/docker-py/blob/
73434476b32136b136e1cdb0913fd123126f2a52/
docker/client.py#L111-L114
"""
ports = container.get('ports')
if ports is None:
return
return [port if not isinstance(port, list) else tuple(port)
for port in ports]
def exec_with_retries(
self, func, exceptions, message, retries=0, interval=0):
# TODO(eli): refactor it and make retries
# as a decorator
intervals = retries * [interval]
for interval in intervals:
try:
return func()
except exceptions as exc:
if str(exc).endswith(message):
time.sleep(interval)
continue
raise
return func()
def get_container_links(self, container):
links = []
if container.get('links'):
for container_link in container.get('links'):
link_container = self.container_by_id(
container_link['id'])
links.append((
link_container['container_name'],
container_link['alias']))
return links
@classmethod
def build_dependencies_graph(cls, containers):
"""Builds graph which based on
`volumes_from` and `link` parameters
of container.
:returns: dict where keys are nodes and
values are lists of dependencies
"""
graph = {}
for container in containers:
graph[container['id']] = sorted(set(
container.get('volumes_from', []) +
[link['id'] for link in container.get('links', [])]))
return graph
def generate_configs(self, autostart=True):
"""Generates supervisor configs
and saves them to configs directory
"""
configs = []
for container in self.new_release_containers:
params = {
'config_name': container['id'],
'service_name': self.make_service_name(container['id']),
'command': u'docker start -a {0}'.format(
container['container_name']),
'autostart': autostart
}
if container['supervisor_config']:
configs.append(params)
self.supervisor.generate_configs(configs)
def make_service_name(self, container_name):
return 'docker-{0}'.format(container_name)
def switch_to_new_configs(self):
"""Switches supervisor to new configs
"""
self.supervisor.switch_to_new_configs()
def volumes_dependencies(self, container):
"""Get list of `volumes` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'volumes_from')
def link_dependencies(self, container):
"""Get list of `link` dependencies
:param contaienr: dict with information about container
"""
return self.dependencies_names(container, 'link')
def dependencies_names(self, container, key):
"""Returns list of dependencies for specified key
:param contaienr: dict with information about container
:param key: key which will be used for dependencies retrieving
:returns: list of container names
"""
names = []
if container.get(key):
for container_id in container.get(key):
container = self.container_by_id(container_id)
names.append(container['container_name'])
return names
def stop_fuel_containers(self):
"""Use docker API to shutdown containers
"""
containers = self.docker_client.containers(limit=-1)
containers_to_stop = filter(
lambda c: c['Image'].startswith(self.config.image_prefix),
containers)
for container in containers_to_stop:
logger.debug(u'Stop container: %s', container)
self.stop_container(container['Id'])
def _get_docker_container_public_ports(self, containers):
"""Returns public ports
:param containers: list of dicts with information about
containers which have `Ports` list
with items where exist `PublicPort`
field
:returns: list of public ports
"""
container_ports = []
for container in containers:
container_ports.extend(container['Ports'])
return [container_port['PublicPort']
for container_port in container_ports]
def stop_container(self, container_id):
"""Stop docker container
:param container_id: container id
"""
logger.debug(u'Stop container: %s', container_id)
try:
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
except requests.exceptions.Timeout:
# NOTE(eli): docker use SIGTERM signal
# to stop container if timeout expired
# docker use SIGKILL to stop container.
# Here we just want to make sure that
# container was stopped.
logger.warn(
u'Couldn\'t stop ctonainer, try '
'to stop it again: %s', container_id)
self.docker_client.stop(
container_id, self.config.docker['stop_container_timeout'])
def start_container(self, container, **params):
"""Start containers
:param container: container name
:param params: dict of arguments for container starting
"""
logger.debug(u'Start container "%s": %s', container['Id'], params)
self.docker_client.start(container['Id'], **params)
def create_container(self, image_name, **params):
"""Create container
:param image_name: name of image
:param params: parameters format equals to
create_container call of docker
client
"""
# We have to delete container because we cannot
# have several containers with the same name
container_name = params.get('name')
if container_name is not None:
self._delete_container_if_exist(container_name)
new_params = deepcopy(params)
new_params['ports'] = self.get_ports(new_params)
logger.debug(u'Create container from image %s: %s',
image_name, new_params)
def func_create():
return self.docker_client.create_container(
image_name,
**new_params)
return self.exec_with_retries(
func_create,
docker.errors.APIError,
"Can't set cookie",
retries=3,
interval=2)
def make_new_release_containers_list(self):
"""Returns list of dicts with information
for new containers.
"""
new_containers = []
for container in self.config.containers:
new_container = deepcopy(container)
new_container['image_name'] = self.make_image_name(
container['from_image'])
new_container['container_name'] = self.make_container_name(
container['id'])
new_containers.append(new_container)
return new_containers
def make_container_name(self, container_id, version=None):
"""Returns container name
:params container_id: container's id
:returns: name of the container
"""
if version is None:
version = self.config.new_version
return u'{0}{1}-{2}'.format(
self.config.container_prefix, version, container_id)
def make_image_name(self, image_id):
"""Makes full image name
:param image_id: image id from config file
:returns: full name
"""
return u'{0}{1}_{2}'.format(
self.config.image_prefix,
image_id,
self.config.new_version)
def container_by_id(self, container_id):
"""Get container from new release by id
:param container_id: id of container
"""
filtered_containers = filter(
lambda c: c['id'] == container_id,
self.new_release_containers)
if not filtered_containers:
raise errors.CannotFindContainerError(
'Cannot find container with id {0}'.format(container_id))
return filtered_containers[0]
def container_docker_id(self, name):
"""Returns running container with specified name
:param name: name of the container
:returns: id of the container or None if not found
:raises CannotFindContainerError:
"""
containers_with_name = self._get_containers_by_name(name)
running_containers = filter(
lambda c: c['Status'].startswith('Up'),
containers_with_name)
if not running_containers:
raise errors.CannotFindContainerError(
'Cannot find running container with name "{0}"'.format(name))
return running_containers[0]['Id']
def _delete_container_if_exist(self, container_name):
"""Deletes docker container if it exists
:param container_name: name of container
"""
found_containers = self._get_containers_by_name(container_name)
for container in found_containers:
self.stop_container(container['Id'])
logger.debug(u'Delete container %s', container)
# TODO(eli): refactor it and make retries
# as a decorator
def func_remove():
self.docker_client.remove_container(container['Id'])
self.exec_with_retries(
func_remove,
docker.errors.APIError,
'Error running removeDevice',
retries=3,
interval=2)
def _get_containers_by_name(self, container_name):
return filter(
lambda c: u'/{0}'.format(container_name) in c['Names'],
self.docker_client.containers(all=True))
def _delete_containers_for_image(self, image):
"""Deletes docker containers for specified image
:param image: name of image
"""
all_containers = self.docker_client.containers(all=True)
containers = filter(
# NOTE(eli): We must use convertation to
# str because in some cases Image is integer
lambda c: str(c.get('Image')).startswith(image),
all_containers)
for container in containers:
logger.debug(u'Try to stop container %s which '
'depends on image %s', container['Id'], image)
self.docker_client.stop(container['Id'])
logger.debug(u'Delete container %s which '
'depends on image %s', container['Id'], image)
self.docker_client.remove_container(container['Id'])
class UpgradeManager(object):
"""Upgrade manager is used to orchestrate upgrading process.
:param upgraders: a list with upgrader classes to use; each upgrader
must inherit the :class:`BaseUpgrader`
:param no_rollback: call :meth:`BaseUpgrader.rollback` method
in case of exception during execution
"""
def __init__(self, upgraders, config, no_rollback=True):
#: an object with configuration context
self._config = config
#: a list of upgraders to use
self._upgraders = upgraders
#: a list of used upgraders (needs by rollback feature)
self._used_upgraders = []
#: should we make rollback in case of error?
self._rollback = not no_rollback
#: version.yaml manager
self._version_file = VersionFile(self._config)
self._version_file.save_current()
def run(self):
"""Runs consequentially all registered upgraders.
.. note:: in case of exception the `rollback` method will be called
"""
logger.info('*** START UPGRADING')
self._version_file.switch_to_new()
for upgrader in self._upgraders:
logger.debug('%s: backuping...', upgrader.__class__.__name__)
try:
upgrader.backup()
except Exception as exc:
logger.exception('%s: failed to backup: "%s"',
upgrader.__class__.__name__, exc)
logger.error('*** UPGRADE FAILED')
raise
for upgrader in self._upgraders:
logger.debug('%s: upgrading...', upgrader.__class__.__name__)
self._used_upgraders.append(upgrader)
try:
upgrader.upgrade()
except Exception as exc:
logger.exception('%s: failed to upgrade: "%s"',
upgrader.__class__.__name__, exc)
if self._rollback:
self.rollback()
logger.error('*** UPGRADE FAILED')
raise
try:
self._on_success()
except Exception as exc:
logger.exception('Could not complete on_success actions due to %s',
six.text_type(exc))
logger.info('*** UPGRADING MASTER NODE DONE SUCCESSFULLY')
logger.info('*** PLEASE REBOOT YOUR BOOTSTRAP NODES IN ORDER TO MAKE'
' SURE THAT THEY USE THE LATEST BOOTSTRAP IMAGE')
def _on_success(self):
"""Do some useful job if upgrade was done successfully.
"""
# Remove saved version files for all upgrades
#
# NOTE(eli): It solves several problems:
#
# 1. user runs upgrade 5.0 -> 5.1 which fails
# upgrade system saves version which we upgrade
# from in file working_dir/5.1/version.yaml.
# Then user runs upgrade 5.0 -> 5.0.1 which
# successfully upgraded. Then user runs again
# upgrade 5.0.1 -> 5.1, but there is saved file
# working_dir/5.1/version.yaml which contains
# 5.0 version, and upgrade system thinks that
# it's upgrading from 5.0 version, as result
# it tries to make database dump from wrong
# version of container.
#
# 2. without this hack user can run upgrade
# second time and loose his data, this hack
# prevents this case because before upgrade
# checker will use current version instead
# of saved version to determine version which
# we run upgrade from.
for version_file in glob.glob(self._config.version_files_mask):
utils.remove(version_file)
self._setup_update_repos()
def _setup_update_repos(self):
"""Setup updates/security repos on master node.
"""
template = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'templates', 'nailgun.repo')
for repo in self._config.master_node_repos:
destination = '{0}.repo'.format(
os.path.join('/etc/yum.repos.d', repo['name']))
utils.render_template_to_file(template, destination, repo)
logger.warning(messages.update_your_master_node)
def rollback(self):
logger.debug('Run rollback')
# because of issue #1452378 [1], we have to perform HostSystem's
# rollback before others. so, move it to the end of list.
#
# [1]: https://bugs.launchpad.net/fuel/+bug/1452378
hostsystem = next((upgrader for upgrader in self._used_upgraders
if isinstance(upgrader, HostSystemUpgrader)), None)
if hostsystem is not None:
self._used_upgraders.remove(hostsystem)
self._used_upgraders.append(hostsystem)
# do rollback in reverse order for all the upgraders
while self._used_upgraders:
upgrader = self._used_upgraders.pop()
logger.debug('%s: rollbacking...', upgrader.__class__.__name__)
upgrader.rollback()
self._version_file.switch_to_previous()
class HostSystemUpgrader(UpgradeEngine):
"""Upgrader for master node host system.
Required for upgrading of packages which
are not under docker, for example fuelclient,
dockerctl.
* add local repo with new packages
* run puppet apply
"""
templates_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../templates'))
def __init__(self, *args, **kwargs):
super(HostSystemUpgrader, self).__init__(*args, **kwargs)
#: host system upgarder specific configs
self.host_system_config = self.config.host_system
#: path to puppet manifests
self.manifest_path = self.host_system_config['manifest_path']
#: path to puppet modules
self.puppet_modules_path = self.host_system_config[
'puppet_modules_path']
#: path to repo template
self.repo_template_path = os.path.join(
self.templates_dir, 'nailgun.repo')
#: new version of fuel
self.version = self.config.new_version
#: path to repository config
self.repo_config_path = self.host_system_config['repo_config_path']
#: path to local repository
self.repo_path = self.host_system_config['repo_path']
#: version file manager
self.version_file = VersionFile(self.config)
self.version_file.save_current()
@property
def required_free_space(self):
"""Required free space to run upgrade
Requires only several megabytes for
repo config.
:returns: dict where key is path to directory
and value is required free space
"""
return {self.repo_config_path: 10}
def upgrade(self):
"""Run host system upgrade process
"""
self.version_file.switch_to_new()
self.update_repo()
self.run_puppet()
def rollback(self):
"""The only thing which we can rollback here
is yum config
"""
self.version_file.switch_to_previous()
self.remove_repo_config()
def on_success(self):
"""Do nothing for this engine
"""
def update_repo(self):
"""Add new centos repository
"""
utils.render_template_to_file(
self.repo_template_path,
self.repo_config_path,
{'version': self.version,
'repo_path': self.repo_path})
def run_puppet(self):
"""Run puppet to upgrade host system
"""
utils.exec_cmd(
'puppet apply -d -v '
'{0} --modulepath={1}'.format(
self.manifest_path, self.puppet_modules_path))
def remove_repo_config(self):
"""Remove yum repository config
"""
utils.remove_if_exists(self.repo_config_path)
def setUp(self):
self.version_file = VersionFile(self.fake_config)
def setUp(self):
self.version_file = VersionFile(self.fake_config)