def ovs_3671_validate_archiving_of_existing_arakoon_data_on_create_test():
        """
        Validate arakoon archiving on extending a cluster with already existing data
        """
        first_sr = GeneralStorageRouter.get_storage_routers()[0]

        cluster_name = 'OVS_3671-single-node-cluster'
        cluster_basedir = '/var/tmp'

        root_client = SSHClient(first_sr, username='******')
        for directory in ['/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon']:
            root_client.dir_create(os.path.dirname(directory))
            root_client.dir_chmod(os.path.dirname(directory), 0755, recursive=True)
            root_client.dir_chown(os.path.dirname(directory), 'ovs', 'ovs', recursive=True)

        files_to_create = ['/'.join([cluster_basedir, 'arakoon', cluster_name, 'db', 'one.db']),
                           '/'.join([cluster_basedir, 'arakoon', cluster_name, 'tlogs', 'one.tlog'])]

        client = SSHClient(first_sr, username='******')
        for filename in files_to_create:
            client.dir_create(os.path.dirname(filename))
            client.dir_chmod(os.path.dirname(filename), 0755, recursive=True)
            client.dir_chown(os.path.dirname(filename), 'ovs', 'ovs', recursive=True)

        client.file_create(files_to_create)
        for filename in files_to_create:
            assert client.file_exists(filename) is True, 'File {0} not present'.format(filename)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('setup and validate single node cluster')
        create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, first_sr.ip,
                                                      cluster_basedir, filesystem=False)
        TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
        ArakoonInstaller.start_cluster(cluster_name, first_sr.ip, False)
        ArakoonInstaller.claim_cluster(cluster_name, first_sr, False, metadata=create_info['metadata'])
        TestArakoon.validate_arakoon_config_files([first_sr], cluster_name)
        TestArakoon.verify_arakoon_structure(root_client, cluster_name, True, True)
        for filename in files_to_create:
            assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('remove cluster')
        ArakoonInstaller.delete_cluster(cluster_name, first_sr.ip, False)
        for filename in files_to_create:
            assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
        TestArakoon.verify_arakoon_structure(root_client, cluster_name, False, False)
Exemplo n.º 2
0
    def _voldrv_arakoon_checkup(create_cluster):
        def add_service(service_storagerouter, arakoon_ports):
            """
            Add a service to the storage router
            :param service_storagerouter: Storage Router to add the service to
            :type service_storagerouter: StorageRouter
            :param arakoon_ports: Port information
            :type arakoon_ports: list
            :return: The newly created and added service
            :rtype: Service
            """
            new_service = Service()
            new_service.name = service_name
            new_service.type = service_type
            new_service.ports = arakoon_ports
            new_service.storagerouter = service_storagerouter
            new_service.save()
            return new_service

        service_name = 'arakoon-voldrv'
        service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)

        current_ips = []
        current_services = []
        for service in service_type.services:
            if service.name == service_name:
                current_services.append(service)
                if service.is_internal is True:
                    current_ips.append(service.storagerouter.ip)

        all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
        available_storagerouters = {}
        for storagerouter in StorageRouterList.get_masters():
            storagerouter.invalidate_dynamics(['partition_config'])
            if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
                available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
            all_sr_ips.append(storagerouter.ip)

        if create_cluster is True and len(current_services) == 0:  # Create new cluster
            metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
            if metadata is None:  # No externally managed cluster found, we create 1 ourselves
                if not available_storagerouters:
                    raise RuntimeError('Could not find any Storage Router with a DB role')

                storagerouter, partition = available_storagerouters.items()[0]
                result = ArakoonInstaller.create_cluster(cluster_name='voldrv',
                                                         cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
                                                         ip=storagerouter.ip,
                                                         base_dir=partition.folder,
                                                         filesystem=False)
                ports = [result['client_port'], result['messaging_port']]
                metadata = result['metadata']
                ArakoonInstaller.restart_cluster_add(cluster_name='voldrv',
                                                     current_ips=current_ips,
                                                     new_ip=storagerouter.ip,
                                                     filesystem=False)
                ArakoonInstaller.claim_cluster(cluster_name='voldrv',
                                               master_ip=storagerouter.ip,
                                               filesystem=False,
                                               metadata=metadata)
                current_ips.append(storagerouter.ip)
            else:
                ports = []
                storagerouter = None

            cluster_name = metadata['cluster_name']
            Configuration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name)
            StorageDriverController._logger.info('Claiming {0} managed arakoon cluster: {1}'.format('externally' if storagerouter is None else 'internally', cluster_name))
            StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
            current_services.append(add_service(service_storagerouter=storagerouter, arakoon_ports=ports))

        cluster_name = Configuration.get('/ovs/framework/arakoon_clusters').get('voldrv')
        if cluster_name is None:
            return
        metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
        if 0 < len(current_services) < len(available_storagerouters) and metadata['internal'] is True:
            for storagerouter, partition in available_storagerouters.iteritems():
                if storagerouter.ip in current_ips:
                    continue
                result = ArakoonInstaller.extend_cluster(master_ip=current_services[0].storagerouter.ip,
                                                         new_ip=storagerouter.ip,
                                                         cluster_name=cluster_name,
                                                         base_dir=partition.folder)
                add_service(storagerouter, [result['client_port'], result['messaging_port']])
                current_ips.append(storagerouter.ip)
                ArakoonInstaller.restart_cluster_add(cluster_name=cluster_name,
                                                     current_ips=current_ips,
                                                     new_ip=storagerouter.ip,
                                                     filesystem=False)
            StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
Exemplo n.º 3
0
    def test_cluster_maintenance(self):
        """
        Validates whether a cluster can be correctly created
        """
        Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]})
        Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]})

        structure = Helper.build_service_structure(
            {'storagerouters': [1, 2]}
        )
        storagerouters = structure['storagerouters']
        System._machine_id = {storagerouters[1].ip: '1',
                              storagerouters[2].ip: '2'}

        # Create new cluster
        mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
        if os.path.exists(mountpoint) and mountpoint != '/':
            shutil.rmtree(mountpoint)
        base_dir = mountpoint + '/test_create_cluster'
        info = ArakoonInstaller.create_cluster('test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir)

        reality = Helper.extract_dir_structure(base_dir)
        expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
                                                                              'files': []},
                                                                    'db': {'dirs': {},
                                                                           'files': []}},
                                                           'files': []}},
                                         'files': []}},
                    'files': []}
        self.assertDictEqual(reality, expected)
        expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1', 'test', ''),
                                           ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                                               '1', storagerouters[1].ip, 10000, base_dir, '1', 10001
                                           ))
        self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
        # @TODO: assert service availability here. It should be stopped

        ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False)
        # @TODO: assert the service is running

        config = ArakoonClusterConfig('test', filesystem=False)
        config.load_config(storagerouters[1].ip)
        client = ArakoonInstaller.build_client(config)
        reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
        self.assertEqual(reality, expected)
        self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))

        ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata'])

        reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
        expected = {'cluster_name': 'test',
                    'cluster_type': 'FWK',
                    'in_use': True,
                    'internal': True}
        self.assertDictEqual(reality, expected)

        # Extending cluster
        mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
        if os.path.exists(mountpoint) and mountpoint != '/':
            shutil.rmtree(mountpoint)
        base_dir2 = mountpoint + '/test_extend_cluster'
        ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2)
        reality = Helper.extract_dir_structure(base_dir)
        expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
                                                                              'files': []},
                                                                    'db': {'dirs': {},
                                                                           'files': []}},
                                                           'files': []}},
                                         'files': []}},
                    'files': []}
        self.assertDictEqual(reality, expected)
        expected = '{0}\n\n{1}\n\n{2}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1,2', 'test', ''),
                                                  ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                                                      '1', storagerouters[1].ip, 10000, base_dir, '1', 10001
                                                  ),
                                                  ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                                                      '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
                                                  ))
        self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
        # @TODO: assert service availability here. It should be stopped

        catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
        SSHClient._run_returns[catchup_command] = None
        SSHClient._run_recordings = []
        ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False)
        self.assertIn(catchup_command, SSHClient._run_recordings)
        # @TODO: assert the service is running

        config = ArakoonClusterConfig('test', filesystem=False)
        config.load_config(storagerouters[2].ip)
        client = ArakoonInstaller.build_client(config)
        reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
        self.assertEqual(reality, expected)

        reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
        expected = {'cluster_name': 'test',
                    'cluster_type': 'FWK',
                    'in_use': True,
                    'internal': True}
        self.assertDictEqual(reality, expected)

        # Shrinking cluster
        ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test')
        reality = Helper.extract_dir_structure(base_dir)
        expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {}, 'files': []}},
                                         'files': []}},
                    'files': []}
        self.assertDictEqual(reality, expected)
        expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('2', 'test', ''),
                                           ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                                               '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
                                           ))
        self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
        # @TODO: assert service availability here. It should be stopped

        ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False)
        # @TODO: assert the service is running

        config = ArakoonClusterConfig('test', filesystem=False)
        config.load_config(storagerouters[2].ip)
        client = ArakoonInstaller.build_client(config)
        reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
        self.assertEqual(reality, expected)

        reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
        expected = {'cluster_name': 'test',
                    'cluster_type': 'FWK',
                    'in_use': True,
                    'internal': True}
        self.assertDictEqual(reality, expected)
    def ovs_3671_validate_archiving_of_existing_arakoon_data_on_create_and_extend_test():
        """
        Validate arakoon archiving when creating and extending an arakoon cluster
        """
        storagerouters = GeneralStorageRouter.get_storage_routers()
        storagerouters.sort(key=lambda k: k.ip)
        if len(storagerouters) < 2:
            TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters)))
            return

        cluster_name = 'OVS_3671-multi-node-cluster'
        cluster_basedir = '/var/tmp'

        archived_files = []
        files_to_create = []
        for index, sr in enumerate(storagerouters):
            root_client = SSHClient(sr, username='******')
            for directory in ['/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon']:
                root_client.dir_create(os.path.dirname(directory))
                root_client.dir_chmod(os.path.dirname(directory), 0755, recursive=True)
                root_client.dir_chown(os.path.dirname(directory), 'ovs', 'ovs', recursive=True)

            files_to_create = ['/'.join([cluster_basedir, 'arakoon', cluster_name, 'db', 'one.db']),
                               '/'.join([cluster_basedir, 'arakoon', cluster_name, 'tlogs', 'one.tlog'])]

            client = SSHClient(sr, username='******')
            for filename in files_to_create:
                client.dir_create(os.path.dirname(filename))
                client.dir_chmod(os.path.dirname(filename), 0755, recursive=True)
                client.dir_chown(os.path.dirname(filename), 'ovs', 'ovs', recursive=True)

            client.file_create(files_to_create)
            for filename in files_to_create:
                assert client.file_exists(filename) is True, 'File {0} not present'.format(filename)

            archived_files = ['/'.join(['/var/log/arakoon', cluster_name, 'archive', 'one.log'])]

            TestArakoon.logger.info('===================================================')
            TestArakoon.logger.info('setup and validate single node cluster')
            if index == 0:
                create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK,
                                                              sr.ip, cluster_basedir, filesystem=False)
                TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
                ArakoonInstaller.start_cluster(cluster_name, sr.ip, False)
                ArakoonInstaller.claim_cluster(cluster_name, sr, False, metadata=create_info['metadata'])
            else:
                ArakoonInstaller.extend_cluster(storagerouters[0].ip, sr.ip, cluster_name, cluster_basedir)
            TestArakoon.validate_arakoon_config_files(storagerouters[:index + 1], cluster_name)
            TestArakoon.verify_arakoon_structure(root_client, cluster_name, True, True)
            TestArakoon.check_archived_directory(client, archived_files)
            for filename in files_to_create:
                assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('remove cluster')
        ArakoonInstaller.delete_cluster(cluster_name, storagerouters[0].ip, False)

        for sr in storagerouters:
            client = SSHClient(sr, username='******')
            TestArakoon.check_archived_directory(client, archived_files)
            for filename in files_to_create:
                assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
            TestArakoon.verify_arakoon_structure(client, cluster_name, False, False)
    def ar_0001_validate_create_extend_shrink_delete_cluster_test():
        """
        Validate extending and shrinking of arakoon clusters
        """
        storagerouters = GeneralStorageRouter.get_storage_routers()
        if not len(storagerouters) >= 3:
            TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters)))
            return

        cluster_name = 'ar_0001'
        cluster_basedir = '/var/tmp/'
        first_sr = storagerouters[0]
        second_sr = storagerouters[1]
        third_sr = storagerouters[2]
        first_root_client = SSHClient(first_sr, username='******')
        second_root_client = SSHClient(second_sr, username='******')
        third_root_client = SSHClient(third_sr, username='******')

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('setup and validate single node cluster')
        create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, first_sr.ip,
                                                      cluster_basedir, filesystem=False)
        TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
        ArakoonInstaller.start_cluster(cluster_name, first_sr.ip, False)
        ArakoonInstaller.claim_cluster(cluster_name, first_sr, False, metadata=create_info['metadata'])
        TestArakoon.validate_arakoon_config_files([first_sr], cluster_name)
        TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('setup and validate two node cluster')
        ArakoonInstaller.extend_cluster(first_sr.ip, second_sr.ip, cluster_name, cluster_basedir)
        TestArakoon.validate_arakoon_config_files([first_sr, second_sr], cluster_name)
        TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
        TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, True)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('setup and validate three node cluster')
        ArakoonInstaller.extend_cluster(first_sr.ip, third_sr.ip, cluster_name, cluster_basedir)
        TestArakoon.validate_arakoon_config_files([first_sr, second_sr, third_sr], cluster_name)

        for client in [first_root_client, second_root_client, third_root_client]:
            TestArakoon.verify_arakoon_structure(client, cluster_name, True, True)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('reduce and validate three node to two node cluster')
        ArakoonInstaller.shrink_cluster(second_sr.ip, first_sr.ip, cluster_name)
        TestArakoon.validate_arakoon_config_files([first_sr, third_sr], cluster_name)
        TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
        TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False)
        TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('reduce and validate two node to one node cluster')
        ArakoonInstaller.shrink_cluster(first_sr.ip, third_sr.ip, cluster_name)
        TestArakoon.validate_arakoon_config_files([third_sr], cluster_name)

        TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, False)
        TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False)
        TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True)

        TestArakoon.logger.info('===================================================')
        TestArakoon.logger.info('remove cluster')
        ArakoonInstaller.delete_cluster(cluster_name, third_sr.ip, False)

        for client in [first_root_client, second_root_client, third_root_client]:
            TestArakoon.verify_arakoon_structure(client, cluster_name, False, False)

        GeneralArakoon.delete_config(cluster_name)
Exemplo n.º 6
0
    def test_cluster_maintenance(self):
        """
        Validates whether a cluster can be correctly created
        """
        Configuration.set('/ovs/framework/hosts/1/ports',
                          {'arakoon': [10000, 10100]})
        Configuration.set('/ovs/framework/hosts/2/ports',
                          {'arakoon': [20000, 20100]})

        structure = Helper.build_service_structure({'storagerouters': [1, 2]})
        storagerouters = structure['storagerouters']
        System._machine_id = {
            storagerouters[1].ip: '1',
            storagerouters[2].ip: '2'
        }

        # Create new cluster
        mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
        if os.path.exists(mountpoint) and mountpoint != '/':
            shutil.rmtree(mountpoint)
        base_dir = mountpoint + '/test_create_cluster'
        info = ArakoonInstaller.create_cluster(
            'test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK,
            storagerouters[1].ip, base_dir)

        reality = Helper.extract_dir_structure(base_dir)
        expected = {
            'dirs': {
                'arakoon': {
                    'dirs': {
                        'test': {
                            'dirs': {
                                'tlogs': {
                                    'dirs': {},
                                    'files': []
                                },
                                'db': {
                                    'dirs': {},
                                    'files': []
                                }
                            },
                            'files': []
                        }
                    },
                    'files': []
                }
            },
            'files': []
        }
        self.assertDictEqual(reality, expected)
        expected = '{0}\n\n{1}\n\n'.format(
            ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
                '1', 'test', ''),
            ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                '1', storagerouters[1].ip, 10000, base_dir, '1', 10001))
        self.assertEqual(
            Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
                              raw=True), expected)
        # @TODO: assert service availability here. It should be stopped

        ArakoonInstaller.start_cluster('test',
                                       storagerouters[1].ip,
                                       filesystem=False)
        # @TODO: assert the service is running

        config = ArakoonClusterConfig('test', filesystem=False)
        config.load_config(storagerouters[1].ip)
        client = ArakoonInstaller.build_client(config)
        reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
        self.assertEqual(reality, expected)
        self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))

        ArakoonInstaller.claim_cluster('test',
                                       storagerouters[1].ip,
                                       filesystem=False,
                                       metadata=info['metadata'])

        reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
        expected = {
            'cluster_name': 'test',
            'cluster_type': 'FWK',
            'in_use': True,
            'internal': True
        }
        self.assertDictEqual(reality, expected)

        # Extending cluster
        mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
        if os.path.exists(mountpoint) and mountpoint != '/':
            shutil.rmtree(mountpoint)
        base_dir2 = mountpoint + '/test_extend_cluster'
        ArakoonInstaller.extend_cluster(storagerouters[1].ip,
                                        storagerouters[2].ip, 'test',
                                        base_dir2)
        reality = Helper.extract_dir_structure(base_dir)
        expected = {
            'dirs': {
                'arakoon': {
                    'dirs': {
                        'test': {
                            'dirs': {
                                'tlogs': {
                                    'dirs': {},
                                    'files': []
                                },
                                'db': {
                                    'dirs': {},
                                    'files': []
                                }
                            },
                            'files': []
                        }
                    },
                    'files': []
                }
            },
            'files': []
        }
        self.assertDictEqual(reality, expected)
        expected = '{0}\n\n{1}\n\n{2}\n\n'.format(
            ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
                '1,2', 'test', ''),
            ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                '1', storagerouters[1].ip, 10000, base_dir, '1', 10001),
            ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
        self.assertEqual(
            Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
                              raw=True), expected)
        # @TODO: assert service availability here. It should be stopped

        catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
        SSHClient._run_returns[catchup_command] = None
        SSHClient._run_recordings = []
        ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip],
                                             storagerouters[2].ip,
                                             filesystem=False)
        self.assertIn(catchup_command, SSHClient._run_recordings)
        # @TODO: assert the service is running

        config = ArakoonClusterConfig('test', filesystem=False)
        config.load_config(storagerouters[2].ip)
        client = ArakoonInstaller.build_client(config)
        reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
        self.assertEqual(reality, expected)

        reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
        expected = {
            'cluster_name': 'test',
            'cluster_type': 'FWK',
            'in_use': True,
            'internal': True
        }
        self.assertDictEqual(reality, expected)

        # Shrinking cluster
        ArakoonInstaller.shrink_cluster(storagerouters[1].ip,
                                        storagerouters[2].ip, 'test')
        reality = Helper.extract_dir_structure(base_dir)
        expected = {
            'dirs': {
                'arakoon': {
                    'dirs': {
                        'test': {
                            'dirs': {},
                            'files': []
                        }
                    },
                    'files': []
                }
            },
            'files': []
        }
        self.assertDictEqual(reality, expected)
        expected = '{0}\n\n{1}\n\n'.format(
            ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
                '2', 'test', ''),
            ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
                '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
        self.assertEqual(
            Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
                              raw=True), expected)
        # @TODO: assert service availability here. It should be stopped

        ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip],
                                                filesystem=False)
        # @TODO: assert the service is running

        config = ArakoonClusterConfig('test', filesystem=False)
        config.load_config(storagerouters[2].ip)
        client = ArakoonInstaller.build_client(config)
        reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
        self.assertEqual(reality, expected)

        reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
        expected = {
            'cluster_name': 'test',
            'cluster_type': 'FWK',
            'in_use': True,
            'internal': True
        }
        self.assertDictEqual(reality, expected)