class TestSupervisorClient(BaseTestCase): def setUp(self): self.utils_patcher = mock.patch( 'fuel_upgrade.clients.supervisor_client.utils') self.utils_mock = self.utils_patcher.start() self.supervisor = SupervisorClient(self.fake_config, '0') type(self.supervisor).supervisor = mock.PropertyMock() self.new_version_supervisor_path = '/etc/supervisord.d/9999' self.previous_version_supervisor_path = '/etc/supervisord.d/0' def tearDown(self): self.utils_patcher.stop() def test_switch_to_new_configs(self, os_mock): self.supervisor.switch_to_new_configs() self.utils_mock.symlink.assert_called_once_with( self.new_version_supervisor_path, self.fake_config.supervisor['current_configs_prefix']) self.supervisor.supervisor.reloadConfig.assert_called_once_with() def test_switch_to_previous_configs(self, os_mock): self.supervisor.switch_to_previous_configs() self.utils_mock.symlink.assert_called_once_with( self.previous_version_supervisor_path, self.fake_config.supervisor['current_configs_prefix']) self.supervisor.supervisor.reloadConfig.assert_called_once_with() def test_stop_all_services(self, _): self.supervisor.stop_all_services() self.supervisor.supervisor.stopAllProcesses.assert_called_once_with() @mock.patch('fuel_upgrade.clients.supervisor_client.SupervisorClient.' 'get_all_processes_safely') def test_restart_and_wait(self, _, __): self.supervisor.restart_and_wait() self.supervisor.supervisor.restart.assert_called_once_with() timeout = self.utils_mock.wait_for_true.call_args[1]['timeout'] self.assertEqual(timeout, 600) # since wait_for_true is mocked in all tests, let's check that # callback really calls get_all_processes_safely function callback = self.utils_mock.wait_for_true.call_args[0][0] callback() self.supervisor.get_all_processes_safely.assert_called_once_with() def test_get_all_processes_safely(self, _): self.supervisor.get_all_processes_safely() self.supervisor.supervisor.getAllProcessInfo.assert_called_once_with() def test_get_all_processes_safely_does_not_raise_error(self, _): for exc in (IOError(), xmlrpclib.Fault('', '')): self.supervisor.supervisor.getAllProcessInfo.side_effect = exc self.assertIsNone(self.supervisor.get_all_processes_safely()) def test_generate_configs(self, _): services = [{ 'config_name': 'config_name1', 'service_name': 'service_name1', 'command': 'cmd1', 'autostart': True }, { 'config_name': 'config_name2', 'service_name': 'service_name2', 'command': 'cmd2', 'autostart': False }] self.supervisor.generate_config = mock.MagicMock() self.supervisor.generate_configs(services) self.assertEqual(self.supervisor.generate_config.call_args_list, [ mock.call('config_name1', 'service_name1', 'cmd1', autostart=True), mock.call('config_name2', 'service_name2', 'cmd2', autostart=False) ]) def test_generate_config(self, _): config_path = '/config/path' with mock.patch('fuel_upgrade.clients.supervisor_client.os.path.join', return_value=config_path): self.supervisor.generate_config('confing_name1', 'docker-service_name1', 'command1') self.utils_mock.render_template_to_file.assert_called_once_with( self.supervisor.supervisor_template_path, config_path, { 'service_name': 'docker-service_name1', 'command': 'command1', 'log_path': '/var/log/docker-service_name1.log', 'autostart': 'true' }) def test_remove_new_configs(self, _): self.supervisor.remove_new_configs() self.utils_mock.remove.assert_called_with('/etc/supervisord.d/9999')
class TestSupervisorClient(BaseTestCase): def setUp(self): self.utils_patcher = mock.patch( 'fuel_upgrade.clients.supervisor_client.utils') self.utils_mock = self.utils_patcher.start() self.supervisor = SupervisorClient(self.fake_config, '0') type(self.supervisor).supervisor = mock.PropertyMock() self.new_version_supervisor_path = '/etc/supervisord.d/9999' self.previous_version_supervisor_path = '/etc/supervisord.d/0' def tearDown(self): self.utils_patcher.stop() def test_switch_to_new_configs(self, os_mock): self.supervisor.switch_to_new_configs() self.utils_mock.symlink.assert_called_once_with( self.new_version_supervisor_path, self.fake_config.supervisor['current_configs_prefix']) self.supervisor.supervisor.reloadConfig.assert_called_once_with() def test_switch_to_previous_configs(self, os_mock): self.supervisor.switch_to_previous_configs() self.utils_mock.symlink.assert_called_once_with( self.previous_version_supervisor_path, self.fake_config.supervisor['current_configs_prefix']) self.supervisor.supervisor.reloadConfig.assert_called_once_with() def test_stop_all_services(self, _): self.supervisor.stop_all_services() self.supervisor.supervisor.stopAllProcesses.assert_called_once_with() @mock.patch('fuel_upgrade.clients.supervisor_client.SupervisorClient.' 'get_all_processes_safely') def test_restart_and_wait(self, _, __): self.supervisor.restart_and_wait() self.supervisor.supervisor.restart.assert_called_once_with() timeout = self.utils_mock.wait_for_true.call_args[1]['timeout'] self.assertEqual(timeout, 600) # since wait_for_true is mocked in all tests, let's check that # callback really calls get_all_processes_safely function callback = self.utils_mock.wait_for_true.call_args[0][0] callback() self.supervisor.get_all_processes_safely.assert_called_once_with() def test_get_all_processes_safely(self, _): self.supervisor.get_all_processes_safely() self.supervisor.supervisor.getAllProcessInfo.assert_called_once_with() def test_get_all_processes_safely_does_not_raise_error(self, _): for exc in (IOError(), xmlrpclib.Fault('', '')): self.supervisor.supervisor.getAllProcessInfo.side_effect = exc self.assertIsNone(self.supervisor.get_all_processes_safely()) def test_generate_configs(self, _): services = [ {'config_name': 'config_name1', 'service_name': 'service_name1', 'command': 'cmd1', 'autostart': True}, {'config_name': 'config_name2', 'service_name': 'service_name2', 'command': 'cmd2', 'autostart': False}] self.supervisor.generate_config = mock.MagicMock() self.supervisor.generate_configs(services) self.assertEqual( self.supervisor.generate_config.call_args_list, [mock.call('config_name1', 'service_name1', 'cmd1', autostart=True), mock.call('config_name2', 'service_name2', 'cmd2', autostart=False)]) def test_generate_config(self, _): config_path = '/config/path' with mock.patch('fuel_upgrade.clients.supervisor_client.os.path.join', return_value=config_path): self.supervisor.generate_config( 'confing_name1', 'docker-service_name1', 'command1') self.utils_mock.render_template_to_file.assert_called_once_with( self.supervisor.supervisor_template_path, config_path, {'service_name': 'docker-service_name1', 'command': 'command1', 'log_path': '/var/log/docker-service_name1.log', 'autostart': 'true'}) def test_remove_new_configs(self, _): self.supervisor.remove_new_configs() self.utils_mock.remove.assert_called_with('/etc/supervisord.d/9999')
class DockerUpgrader(UpgradeEngine): """Docker management system for upgrades """ def __init__(self, *args, **kwargs): super(DockerUpgrader, self).__init__(*args, **kwargs) self.working_directory = self.config.working_directory utils.create_dir_if_not_exists(self.working_directory) self.docker_client = docker.Client( base_url=self.config.docker['url'], version=self.config.docker['api_version'], timeout=self.config.docker['http_timeout']) self.new_release_containers = self.make_new_release_containers_list() self.cobbler_config_path = self.config.cobbler_config_path.format( working_directory=self.working_directory) self.upgrade_verifier = FuelUpgradeVerify(self.config) self.from_version = self.config.from_version self.supervisor = SupervisorClient(self.config, self.from_version) def backup(self): self.save_db() self.save_cobbler_configs() def upgrade(self): """Method with upgarde logic """ # Point to new supervisor configs and restart supervisor in # order to apply them self.switch_to_new_configs() self.supervisor.restart_and_wait() # Stop docker containers (it's safe, since at this time supervisor's # configs are empty. self.stop_fuel_containers() # Upload new docker images and create containers self.upload_images() self.create_and_start_new_containers() # Generate supervisor configs for new containers and restart # supervisor in order to apply them. Note, supervisor's processes # will be attached to running docker containers automatically. self.generate_configs(autostart=True) self.supervisor.restart_and_wait() # Verify that all services up and running self.upgrade_verifier.verify() def rollback(self): """Method which contains rollback logic """ self.supervisor.switch_to_previous_configs() self.supervisor.stop_all_services() self.stop_fuel_containers() self.supervisor.restart_and_wait() self.supervisor.remove_new_configs() @property def required_free_space(self): """Required free space to run upgrade * space for docker * several megabytes for configs * reserve several megabytes for working directory where we keep postgresql dump and cobbler configs :returns: dict where key is path to directory and value is required free space """ return { self.config.docker['dir']: self._calculate_images_size(), self.config.supervisor['configs_prefix']: 10, self.config.fuel_config_path: 10, self.working_directory: 150} def _calculate_images_size(self): return utils.files_size([self.config.images]) def save_db(self): """Saves postgresql database into the file """ logger.debug('Backup database') pg_dump_path = os.path.join(self.working_directory, 'pg_dump_all.sql') pg_dump_files = utils.VersionedFile(pg_dump_path) pg_dump_tmp_path = pg_dump_files.next_file_name() utils.wait_for_true( lambda: self.make_pg_dump(pg_dump_tmp_path, pg_dump_path), timeout=self.config.db_backup_timeout, interval=self.config.db_backup_interval) valid_dumps = filter(utils.verify_postgres_dump, pg_dump_files.sorted_files()) if valid_dumps: utils.hardlink(valid_dumps[0], pg_dump_path, overwrite=True) map(utils.remove_if_exists, valid_dumps[self.config.keep_db_backups_count:]) else: raise errors.DatabaseDumpError( 'Failed to make database dump, there ' 'are no valid database backup ' 'files, {0}'.format(pg_dump_path)) def make_pg_dump(self, pg_dump_tmp_path, pg_dump_path): """Run postgresql dump in container :param str pg_dump_tmp_path: path to temporary dump file :param str pg_dump_path: path to dump which will be restored in the new container, if this file is exists, it means the user already ran upgrade and for some reasons it failed :returns: True if db was successfully dumped or if dump exists False if container isn't running or dump isn't succeed """ try: container_name = self.make_container_name( 'postgres', self.from_version) self.exec_cmd_in_container( container_name, "su postgres -c 'pg_dumpall --clean' > {0}".format( pg_dump_tmp_path)) except (errors.ExecutedErrorNonZeroExitCode, errors.CannotFindContainerError) as exc: utils.remove_if_exists(pg_dump_tmp_path) if not utils.file_exists(pg_dump_path): logger.debug('Failed to make database dump %s', exc) return False logger.debug( 'Failed to make database dump, ' 'will be used dump from previous run: %s', exc) return True def save_cobbler_configs(self): """Copy config files from container """ container_name = self.make_container_name( 'cobbler', self.from_version) try: utils.exec_cmd('docker cp {0}:{1} {2}'.format( container_name, self.config.cobbler_container_config_path, self.cobbler_config_path)) except errors.ExecutedErrorNonZeroExitCode: utils.rmtree(self.cobbler_config_path) raise self.verify_cobbler_configs() def verify_cobbler_configs(self): """Verify that cobbler config directory contains valid data """ configs = glob.glob( self.config.cobbler_config_files_for_verifier.format( cobbler_config_path=self.cobbler_config_path)) # NOTE(eli): cobbler config directory should # contain at least one file (default.json) if len(configs) < 1: raise errors.WrongCobblerConfigsError( 'Cannot find json files in directory {0}'.format( self.cobbler_config_path)) for config in configs: if not utils.check_file_is_valid_json(config): raise errors.WrongCobblerConfigsError( 'Invalid json config {0}'.format(config)) def upload_images(self): """Uploads images to docker """ logger.info('Start image uploading') if not os.path.exists(self.config.images): logger.warn('Cannot find docker images "%s"', self.config.images) return # NOTE(eli): docker-py binding # doesn't have equal call for # image importing which equals to # `docker load` utils.exec_cmd('docker load -i "{0}"'.format(self.config.images)) def create_and_start_new_containers(self): """Create containers in the right order """ logger.info('Started containers creation') graph = self.build_dependencies_graph(self.new_release_containers) logger.debug('Built dependencies graph %s', graph) containers_to_creation = utils.topological_sorting(graph) logger.debug('Resolved creation order %s', containers_to_creation) for container_id in containers_to_creation: container = self.container_by_id(container_id) logger.debug('Start container %s', container) links = self.get_container_links(container) created_container = self.create_container( container['image_name'], name=container.get('container_name'), volumes=container.get('volumes'), ports=container.get('ports'), detach=False) volumes_from = [] for container_id in container.get('volumes_from', []): volume_container = self.container_by_id(container_id) volumes_from.append(volume_container['container_name']) # NOTE(ikalnitsky): # Conflicting options: --net=host can't be used with links. # Still, we need links at least for resolving containers # start order. if container.get('network_mode') == 'host': links = None self.start_container( created_container, port_bindings=container.get('port_bindings'), links=links, volumes_from=volumes_from, binds=container.get('binds'), network_mode=container.get('network_mode'), privileged=container.get('privileged', False)) if container.get('after_container_creation_command'): self.run_after_container_creation_command(container) def run_after_container_creation_command(self, container): """Runs command in container with retries in case of error :param container: dict with container information """ command = container['after_container_creation_command'] def execute(): self.exec_cmd_in_container(container['container_name'], command) self.exec_with_retries( execute, errors.ExecutedErrorNonZeroExitCode, '', retries=30, interval=4) def exec_cmd_in_container(self, container_name, cmd): """Execute command in running container :param name: name of the container, like fuel-core-5.1-nailgun """ db_container_id = self.container_docker_id(container_name) utils.exec_cmd("dockerctl shell {0} {1}".format(db_container_id, cmd)) def get_ports(self, container): """Docker binding accepts ports as tuple, here we convert from list to tuple. FIXME(eli): https://github.com/dotcloud/docker-py/blob/ 73434476b32136b136e1cdb0913fd123126f2a52/ docker/client.py#L111-L114 """ ports = container.get('ports') if ports is None: return return [port if not isinstance(port, list) else tuple(port) for port in ports] def exec_with_retries( self, func, exceptions, message, retries=0, interval=0): # TODO(eli): refactor it and make retries # as a decorator intervals = retries * [interval] for interval in intervals: try: return func() except exceptions as exc: if str(exc).endswith(message): time.sleep(interval) continue raise return func() def get_container_links(self, container): links = [] if container.get('links'): for container_link in container.get('links'): link_container = self.container_by_id( container_link['id']) links.append(( link_container['container_name'], container_link['alias'])) return links @classmethod def build_dependencies_graph(cls, containers): """Builds graph which based on `volumes_from` and `link` parameters of container. :returns: dict where keys are nodes and values are lists of dependencies """ graph = {} for container in containers: graph[container['id']] = sorted(set( container.get('volumes_from', []) + [link['id'] for link in container.get('links', [])])) return graph def generate_configs(self, autostart=True): """Generates supervisor configs and saves them to configs directory """ configs = [] for container in self.new_release_containers: params = { 'config_name': container['id'], 'service_name': self.make_service_name(container['id']), 'command': 'docker start -a {0}'.format( container['container_name']), 'autostart': autostart } if container['supervisor_config']: configs.append(params) self.supervisor.generate_configs(configs) def make_service_name(self, container_name): return 'docker-{0}'.format(container_name) def switch_to_new_configs(self): """Switches supervisor to new configs """ self.supervisor.switch_to_new_configs() def volumes_dependencies(self, container): """Get list of `volumes` dependencies :param contaienr: dict with information about container """ return self.dependencies_names(container, 'volumes_from') def link_dependencies(self, container): """Get list of `link` dependencies :param contaienr: dict with information about container """ return self.dependencies_names(container, 'link') def dependencies_names(self, container, key): """Returns list of dependencies for specified key :param contaienr: dict with information about container :param key: key which will be used for dependencies retrieving :returns: list of container names """ names = [] if container.get(key): for container_id in container.get(key): container = self.container_by_id(container_id) names.append(container['container_name']) return names def stop_fuel_containers(self): """Use docker API to shutdown containers """ containers = self.docker_client.containers(limit=-1) containers_to_stop = filter( lambda c: c['Image'].startswith(self.config.image_prefix), containers) for container in containers_to_stop: logger.debug('Stop container: %s', container) self.stop_container(container['Id']) def _get_docker_container_public_ports(self, containers): """Returns public ports :param containers: list of dicts with information about containers which have `Ports` list with items where exist `PublicPort` field :returns: list of public ports """ container_ports = [] for container in containers: container_ports.extend(container['Ports']) return [container_port['PublicPort'] for container_port in container_ports] def stop_container(self, container_id): """Stop docker container :param container_id: container id """ logger.debug('Stop container: %s', container_id) try: self.docker_client.stop( container_id, self.config.docker['stop_container_timeout']) except requests.exceptions.Timeout: # NOTE(eli): docker use SIGTERM signal # to stop container if timeout expired # docker use SIGKILL to stop container. # Here we just want to make sure that # container was stopped. logger.warn( 'Couldn\'t stop ctonainer, try ' 'to stop it again: %s', container_id) self.docker_client.stop( container_id, self.config.docker['stop_container_timeout']) def start_container(self, container, **params): """Start containers :param container: container name :param params: dict of arguments for container starting """ logger.debug('Start container "%s": %s', container['Id'], params) self.docker_client.start(container['Id'], **params) def create_container(self, image_name, **params): """Create container :param image_name: name of image :param params: parameters format equals to create_container call of docker client """ # We have to delete container because we cannot # have several containers with the same name container_name = params.get('name') if container_name is not None: self._delete_container_if_exist(container_name) new_params = deepcopy(params) new_params['ports'] = self.get_ports(new_params) logger.debug('Create container from image %s: %s', image_name, new_params) def func_create(): return self.docker_client.create_container( image_name, **new_params) return self.exec_with_retries( func_create, docker.errors.APIError, "Can't set cookie", retries=3, interval=2) def make_new_release_containers_list(self): """Returns list of dicts with information for new containers. """ new_containers = [] for container in self.config.containers: new_container = deepcopy(container) new_container['image_name'] = self.make_image_name( container['from_image']) new_container['container_name'] = self.make_container_name( container['id']) new_containers.append(new_container) return new_containers def make_container_name(self, container_id, version=None): """Returns container name :params container_id: container's id :returns: name of the container """ if version is None: version = self.config.new_version return '{0}{1}-{2}'.format( self.config.container_prefix, version, container_id) def make_image_name(self, image_id): """Makes full image name :param image_id: image id from config file :returns: full name """ return '{0}{1}_{2}'.format( self.config.image_prefix, image_id, self.config.new_version) def container_by_id(self, container_id): """Get container from new release by id :param container_id: id of container """ filtered_containers = filter( lambda c: c['id'] == container_id, self.new_release_containers) if not filtered_containers: raise errors.CannotFindContainerError( 'Cannot find container with id {0}'.format(container_id)) return filtered_containers[0] def container_docker_id(self, name): """Returns running container with specified name :param name: name of the container :returns: id of the container or None if not found :raises CannotFindContainerError: """ containers_with_name = self._get_containers_by_name(name) running_containers = filter( lambda c: c['Status'].startswith('Up'), containers_with_name) if not running_containers: raise errors.CannotFindContainerError( 'Cannot find running container with name "{0}"'.format(name)) return running_containers[0]['Id'] def _delete_container_if_exist(self, container_name): """Deletes docker container if it exists :param container_name: name of container """ found_containers = self._get_containers_by_name(container_name) for container in found_containers: self.stop_container(container['Id']) logger.debug('Delete container %s', container) # TODO(eli): refactor it and make retries # as a decorator def func_remove(): self.docker_client.remove_container(container['Id']) self.exec_with_retries( func_remove, docker.errors.APIError, 'Error running removeDevice', retries=3, interval=2) def _get_containers_by_name(self, container_name): return filter( lambda c: '/{0}'.format(container_name) in c['Names'], self.docker_client.containers(all=True)) def _delete_containers_for_image(self, image): """Deletes docker containers for specified image :param image: name of image """ all_containers = self.docker_client.containers(all=True) containers = filter( # NOTE(eli): We must use convertation to # str because in some cases Image is integer lambda c: str(c.get('Image')).startswith(image), all_containers) for container in containers: logger.debug('Try to stop container %s which ' 'depends on image %s', container['Id'], image) self.docker_client.stop(container['Id']) logger.debug('Delete container %s which ' 'depends on image %s', container['Id'], image) self.docker_client.remove_container(container['Id'])