def test_multi_node(self): base_port = Configuration.get('ovs.ports.arakoon')[0] cluster = 'one' nodes = sorted(TestArakoonInstaller.nodes.keys()) nodes = dict((node, SSHClient(node)) for node in nodes) first_node = nodes.keys()[0] ArakoonInstaller.create_cluster(cluster, first_node, []) for node in nodes[1:]: ArakoonInstaller.extend_cluster(first_node, node, cluster, []) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes)) for node in nodes: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node, client in nodes.iteritems(): contents = client.file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip()) ArakoonInstaller.shrink_cluster(nodes[1], first_node, cluster) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:])) for node in nodes.keys()[1:]: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node, client in nodes.iteritems(): if node == first_node: continue contents = client.file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip())
def test_single_node(self): base_port = Configuration.get('ovs.ports.arakoon')[0] cluster = 'one' node = sorted(TestArakoonInstaller.nodes.keys())[0] ArakoonInstaller.create_cluster(cluster, node, []) contents = SSHClient(node).file_read(self._get_config_path(cluster)) expected = TestArakoonInstaller.expected_global.format(cluster, TestArakoonInstaller.nodes[node]) expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) self.assertEqual(contents.strip(), expected.strip())
def test_single_node(self): node = sorted(TestArakoonInstaller.nodes.keys())[0] result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, node, '/tmp/db') contents = SSHClient(node).file_read(TestArakoonInstaller.cluster_config_file) expected = TestArakoonInstaller.expected_global.format(TestArakoonInstaller.nodes[node], TestArakoonInstaller.cluster_name) expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node) self.assertEqual(contents.strip(), expected.strip())
def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_result): """ Add a service to the storage router :param service_storagerouter: Storage Router to add the service to :type service_storagerouter: StorageRouter :param arakoon_result: Port information :type arakoon_result: Dictionary :return: The newly created and added service """ new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']] new_service.storagerouter = service_storagerouter new_service.save() return new_service cluster_name = 'voldrv' service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name('Arakoon') current_services = [] current_ips = [] for service in service_type.services: if service.name == service_name: current_services.append(service) current_ips.append(service.storagerouter.ip) all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0: storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name=cluster_name, ip=storagerouter.ip, base_dir=partition.folder) current_services.append(add_service(storagerouter, result)) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) current_ips.append(storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver() if 0 < len(current_services) < len(available_storagerouters): for storagerouter, partition in available_storagerouters.iteritems(): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster( current_services[0].storagerouter.ip, storagerouter.ip, cluster_name, partition.folder ) add_service(storagerouter, result) current_ips.append(storagerouter.ip) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver()
def test_single_node(self): node = sorted(TestArakoonInstaller.nodes.keys())[0] result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, node, '/tmp/db') contents = SSHClient(node).file_read(TestArakoonInstaller.cluster_config_file) expected = TestArakoonInstaller.expected_global.format(TestArakoonInstaller.nodes[node], TestArakoonInstaller.cluster_name) expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node) self.assertEqual(contents.strip(), expected.strip())
def ovs_3671_validate_archiving_of_existing_arakoon_data_on_create_test(): """ Validate arakoon archiving on extending a cluster with already existing data """ first_sr = GeneralStorageRouter.get_storage_routers()[0] cluster_name = 'OVS_3671-single-node-cluster' cluster_basedir = '/var/tmp' root_client = SSHClient(first_sr, username='******') for directory in ['/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon']: root_client.dir_create(os.path.dirname(directory)) root_client.dir_chmod(os.path.dirname(directory), 0755, recursive=True) root_client.dir_chown(os.path.dirname(directory), 'ovs', 'ovs', recursive=True) files_to_create = ['/'.join([cluster_basedir, 'arakoon', cluster_name, 'db', 'one.db']), '/'.join([cluster_basedir, 'arakoon', cluster_name, 'tlogs', 'one.tlog'])] client = SSHClient(first_sr, username='******') for filename in files_to_create: client.dir_create(os.path.dirname(filename)) client.dir_chmod(os.path.dirname(filename), 0755, recursive=True) client.dir_chown(os.path.dirname(filename), 'ovs', 'ovs', recursive=True) client.file_create(files_to_create) for filename in files_to_create: assert client.file_exists(filename) is True, 'File {0} not present'.format(filename) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('setup and validate single node cluster') create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, first_sr.ip, cluster_basedir, filesystem=False) TestArakoon.logger.info('create_info: \n{0}'.format(create_info)) ArakoonInstaller.start_cluster(cluster_name, first_sr.ip, False) ArakoonInstaller.claim_cluster(cluster_name, first_sr, False, metadata=create_info['metadata']) TestArakoon.validate_arakoon_config_files([first_sr], cluster_name) TestArakoon.verify_arakoon_structure(root_client, cluster_name, True, True) for filename in files_to_create: assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('remove cluster') ArakoonInstaller.delete_cluster(cluster_name, first_sr.ip, False) for filename in files_to_create: assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename) TestArakoon.verify_arakoon_structure(root_client, cluster_name, False, False)
def test_multi_node(self): nodes = sorted(TestArakoonInstaller.nodes.keys()) nodes = dict((node, SSHClient(node)) for node in nodes) first_node = nodes.keys()[0] result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, first_node, "/tmp/db") for node in nodes.keys()[1:]: ArakoonInstaller.extend_cluster(first_node, node, TestArakoonInstaller.cluster_name, "/tmp/db") expected = TestArakoonInstaller.expected_global.format( ",".join(TestArakoonInstaller.nodes[node] for node in nodes), TestArakoonInstaller.cluster_name ) for node in nodes: expected += TestArakoonInstaller.expected_base.format( TestArakoonInstaller.nodes[node], result["client_port"], result["messaging_port"], TestArakoonInstaller.cluster_name, node, ) expected = expected.strip() for node, client in nodes.iteritems(): contents = client.file_read(TestArakoonInstaller.cluster_config_file) self.assertEqual(contents.strip(), expected.strip()) ArakoonInstaller.shrink_cluster(first_node, TestArakoonInstaller.cluster_name) expected = TestArakoonInstaller.expected_global.format( ",".join(TestArakoonInstaller.nodes[node] for node in nodes[1:]), TestArakoonInstaller.cluster_name ) for node in nodes.keys()[1:]: expected += TestArakoonInstaller.expected_base.format( TestArakoonInstaller.nodes[node], result["client_port"], result["messaging_port"], TestArakoonInstaller.cluster_name, node, ) expected = expected.strip() for node, client in nodes.iteritems(): if node == first_node: continue contents = client.file_read(TestArakoonInstaller.cluster_config_file) self.assertEqual(contents.strip(), expected.strip())
def test_multi_node(self): nodes = sorted(TestArakoonInstaller.nodes.keys()) nodes = dict((node, SSHClient(node)) for node in nodes) first_node = nodes.keys()[0] result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, first_node, '/tmp/db') for node in nodes.keys()[1:]: ArakoonInstaller.extend_cluster(first_node, node, TestArakoonInstaller.cluster_name, '/tmp/db') expected = TestArakoonInstaller.expected_global.format(','.join(TestArakoonInstaller.nodes[node] for node in nodes), TestArakoonInstaller.cluster_name) for node in nodes: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node) expected = expected.strip() for node, client in nodes.iteritems(): contents = client.file_read(TestArakoonInstaller.cluster_config_file) self.assertEqual(contents.strip(), expected.strip()) ArakoonInstaller.shrink_cluster(first_node, TestArakoonInstaller.cluster_name) expected = TestArakoonInstaller.expected_global.format(','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:]), TestArakoonInstaller.cluster_name) for node in nodes.keys()[1:]: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node) expected = expected.strip() for node, client in nodes.iteritems(): if node == first_node: continue contents = client.file_read(TestArakoonInstaller.cluster_config_file) self.assertEqual(contents.strip(), expected.strip())
def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_ports): """ Add a service to the storage router :param service_storagerouter: Storage Router to add the service to :type service_storagerouter: StorageRouter :param arakoon_ports: Port information :type arakoon_ports: list :return: The newly created and added service :rtype: Service """ new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = arakoon_ports new_service.storagerouter = service_storagerouter new_service.save() return new_service service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON) current_ips = [] current_services = [] for service in service_type.services: if service.name == service_name: current_services.append(service) if service.is_internal is True: current_ips.append(service.storagerouter.ip) all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len(current_services) == 0: # Create new cluster metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD) if metadata is None: # No externally managed cluster found, we create 1 ourselves if not available_storagerouters: raise RuntimeError('Could not find any Storage Router with a DB role') storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name='voldrv', cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD, ip=storagerouter.ip, base_dir=partition.folder, filesystem=False) ports = [result['client_port'], result['messaging_port']] metadata = result['metadata'] ArakoonInstaller.restart_cluster_add(cluster_name='voldrv', current_ips=current_ips, new_ip=storagerouter.ip, filesystem=False) ArakoonInstaller.claim_cluster(cluster_name='voldrv', master_ip=storagerouter.ip, filesystem=False, metadata=metadata) current_ips.append(storagerouter.ip) else: ports = [] storagerouter = None cluster_name = metadata['cluster_name'] Configuration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name) StorageDriverController._logger.info('Claiming {0} managed arakoon cluster: {1}'.format('externally' if storagerouter is None else 'internally', cluster_name)) StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name) current_services.append(add_service(service_storagerouter=storagerouter, arakoon_ports=ports)) cluster_name = Configuration.get('/ovs/framework/arakoon_clusters').get('voldrv') if cluster_name is None: return metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name) if 0 < len(current_services) < len(available_storagerouters) and metadata['internal'] is True: for storagerouter, partition in available_storagerouters.iteritems(): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster(master_ip=current_services[0].storagerouter.ip, new_ip=storagerouter.ip, cluster_name=cluster_name, base_dir=partition.folder) add_service(storagerouter, [result['client_port'], result['messaging_port']]) current_ips.append(storagerouter.ip) ArakoonInstaller.restart_cluster_add(cluster_name=cluster_name, current_ips=current_ips, new_ip=storagerouter.ip, filesystem=False) StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def test_cluster_maintenance(self): """ Validates whether a cluster can be correctly created """ Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]}) Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]}) structure = Helper.build_service_structure( {'storagerouters': [1, 2]} ) storagerouters = structure['storagerouters'] System._machine_id = {storagerouters[1].ip: '1', storagerouters[2].ip: '2'} # Create new cluster mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir = mountpoint + '/test_create_cluster' info = ArakoonInstaller.create_cluster('test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir) reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {}, 'files': []}, 'db': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[1].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY)) ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata']) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected) # Extending cluster mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir2 = mountpoint + '/test_extend_cluster' ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2) reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {}, 'files': []}, 'db': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n{2}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1,2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001 ), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only' SSHClient._run_returns[catchup_command] = None SSHClient._run_recordings = [] ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False) self.assertIn(catchup_command, SSHClient._run_recordings) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected) # Shrinking cluster ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test') reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected)
def ovs_3671_validate_archiving_of_existing_arakoon_data_on_create_and_extend_test(): """ Validate arakoon archiving when creating and extending an arakoon cluster """ storagerouters = GeneralStorageRouter.get_storage_routers() storagerouters.sort(key=lambda k: k.ip) if len(storagerouters) < 2: TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters))) return cluster_name = 'OVS_3671-multi-node-cluster' cluster_basedir = '/var/tmp' archived_files = [] files_to_create = [] for index, sr in enumerate(storagerouters): root_client = SSHClient(sr, username='******') for directory in ['/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon']: root_client.dir_create(os.path.dirname(directory)) root_client.dir_chmod(os.path.dirname(directory), 0755, recursive=True) root_client.dir_chown(os.path.dirname(directory), 'ovs', 'ovs', recursive=True) files_to_create = ['/'.join([cluster_basedir, 'arakoon', cluster_name, 'db', 'one.db']), '/'.join([cluster_basedir, 'arakoon', cluster_name, 'tlogs', 'one.tlog'])] client = SSHClient(sr, username='******') for filename in files_to_create: client.dir_create(os.path.dirname(filename)) client.dir_chmod(os.path.dirname(filename), 0755, recursive=True) client.dir_chown(os.path.dirname(filename), 'ovs', 'ovs', recursive=True) client.file_create(files_to_create) for filename in files_to_create: assert client.file_exists(filename) is True, 'File {0} not present'.format(filename) archived_files = ['/'.join(['/var/log/arakoon', cluster_name, 'archive', 'one.log'])] TestArakoon.logger.info('===================================================') TestArakoon.logger.info('setup and validate single node cluster') if index == 0: create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, sr.ip, cluster_basedir, filesystem=False) TestArakoon.logger.info('create_info: \n{0}'.format(create_info)) ArakoonInstaller.start_cluster(cluster_name, sr.ip, False) ArakoonInstaller.claim_cluster(cluster_name, sr, False, metadata=create_info['metadata']) else: ArakoonInstaller.extend_cluster(storagerouters[0].ip, sr.ip, cluster_name, cluster_basedir) TestArakoon.validate_arakoon_config_files(storagerouters[:index + 1], cluster_name) TestArakoon.verify_arakoon_structure(root_client, cluster_name, True, True) TestArakoon.check_archived_directory(client, archived_files) for filename in files_to_create: assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('remove cluster') ArakoonInstaller.delete_cluster(cluster_name, storagerouters[0].ip, False) for sr in storagerouters: client = SSHClient(sr, username='******') TestArakoon.check_archived_directory(client, archived_files) for filename in files_to_create: assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename) TestArakoon.verify_arakoon_structure(client, cluster_name, False, False)
def ar_0001_validate_create_extend_shrink_delete_cluster_test(): """ Validate extending and shrinking of arakoon clusters """ storagerouters = GeneralStorageRouter.get_storage_routers() if not len(storagerouters) >= 3: TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters))) return cluster_name = 'ar_0001' cluster_basedir = '/var/tmp/' first_sr = storagerouters[0] second_sr = storagerouters[1] third_sr = storagerouters[2] first_root_client = SSHClient(first_sr, username='******') second_root_client = SSHClient(second_sr, username='******') third_root_client = SSHClient(third_sr, username='******') TestArakoon.logger.info('===================================================') TestArakoon.logger.info('setup and validate single node cluster') create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, first_sr.ip, cluster_basedir, filesystem=False) TestArakoon.logger.info('create_info: \n{0}'.format(create_info)) ArakoonInstaller.start_cluster(cluster_name, first_sr.ip, False) ArakoonInstaller.claim_cluster(cluster_name, first_sr, False, metadata=create_info['metadata']) TestArakoon.validate_arakoon_config_files([first_sr], cluster_name) TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('setup and validate two node cluster') ArakoonInstaller.extend_cluster(first_sr.ip, second_sr.ip, cluster_name, cluster_basedir) TestArakoon.validate_arakoon_config_files([first_sr, second_sr], cluster_name) TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True) TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, True) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('setup and validate three node cluster') ArakoonInstaller.extend_cluster(first_sr.ip, third_sr.ip, cluster_name, cluster_basedir) TestArakoon.validate_arakoon_config_files([first_sr, second_sr, third_sr], cluster_name) for client in [first_root_client, second_root_client, third_root_client]: TestArakoon.verify_arakoon_structure(client, cluster_name, True, True) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('reduce and validate three node to two node cluster') ArakoonInstaller.shrink_cluster(second_sr.ip, first_sr.ip, cluster_name) TestArakoon.validate_arakoon_config_files([first_sr, third_sr], cluster_name) TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True) TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False) TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('reduce and validate two node to one node cluster') ArakoonInstaller.shrink_cluster(first_sr.ip, third_sr.ip, cluster_name) TestArakoon.validate_arakoon_config_files([third_sr], cluster_name) TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, False) TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False) TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True) TestArakoon.logger.info('===================================================') TestArakoon.logger.info('remove cluster') ArakoonInstaller.delete_cluster(cluster_name, third_sr.ip, False) for client in [first_root_client, second_root_client, third_root_client]: TestArakoon.verify_arakoon_structure(client, cluster_name, False, False) GeneralArakoon.delete_config(cluster_name)
def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_result): new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']] new_service.storagerouter = service_storagerouter new_service.save() return new_service cluster_name = 'voldrv' service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name('Arakoon') current_services = [] current_ips = [] for service in service_type.services: if service.name == service_name: current_services.append(service) current_ips.append(service.storagerouter.ip) all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0: storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name=cluster_name, ip=storagerouter.ip, exclude_ports=ServiceList.get_ports_for_ip(storagerouter.ip), base_dir=partition.folder) current_services.append(add_service(storagerouter, result)) for sr_ip in all_sr_ips: if sr_ip not in current_ips: ArakoonInstaller.deploy_to_slave(storagerouter.ip, sr_ip, cluster_name) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) current_ips.append(storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver() if 0 < len(current_services) < len(available_storagerouters): distributed = False for storagerouter, partition in available_storagerouters.iteritems(): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster( current_services[0].storagerouter.ip, storagerouter.ip, cluster_name, ServiceList.get_ports_for_ip(storagerouter.ip), partition.folder ) add_service(storagerouter, result) current_ips.append(storagerouter.ip) if distributed is False: distributed = True for sr_ip in all_sr_ips: if sr_ip not in current_ips: ArakoonInstaller.deploy_to_slave(current_services[0].storagerouter.ip, sr_ip, cluster_name) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver()
def test_node_config_checkup(self): """ Validates correct working of cluster registry checkup """ base_structure = {'1': {'vrouter_id': '1', 'message_host': '10.0.1.1', 'message_port': 1, 'xmlrpc_host': '10.0.0.1', 'xmlrpc_port': 2, 'failovercache_host': '10.0.1.1', 'failovercache_port': 3, 'network_server_uri': 'tcp://10.0.1.1:4', 'node_distance_map': None}, '2': {'vrouter_id': '2', 'message_host': '10.0.1.2', 'message_port': 1, 'xmlrpc_host': '10.0.0.2', 'xmlrpc_port': 2, 'failovercache_host': '10.0.1.2', 'failovercache_port': 3, 'network_server_uri': 'tcp://10.0.1.2:4', 'node_distance_map': None}} def _validate_node_config(_config, _expected_map): expected = copy.deepcopy(base_structure[_config.vrouter_id]) expected['node_distance_map'] = _expected_map[_config.vrouter_id] self.assertDictEqual(expected, {'vrouter_id': _config.vrouter_id, 'message_host': _config.message_host, 'message_port': _config.message_port, 'xmlrpc_host': _config.xmlrpc_host, 'xmlrpc_port': _config.xmlrpc_port, 'failovercache_host': _config.failovercache_host, 'failovercache_port': _config.failovercache_port, 'network_server_uri': _config.network_server_uri, 'node_distance_map': _config.node_distance_map}) structure = Helper.build_service_structure( {'vpools': [1], 'domains': [1, 2], 'storagerouters': [1, 2], 'storagedrivers': [(1, 1, 1), (2, 1, 2)], # (<id>, <vpool_id>, <storagerouter_id>) 'storagerouter_domains': [(1, 1, 1, False), (2, 2, 1, False)]} # (id>, <storagerouter_id>, <domain_id>, <backup>) ) storagerouters = structure['storagerouters'] vpool = structure['vpools'][1] System._machine_id = {storagerouters[1].ip: '1', storagerouters[2].ip: '2'} ArakoonInstaller.create_cluster('voldrv', ServiceType.ARAKOON_CLUSTER_TYPES.SD, storagerouters[1].ip, '/tmp') # Initial run, it will now be configured StorageRouterClient.node_config_recordings = [] result = StorageDriverController.cluster_registry_checkup() self.assertDictEqual(result, {vpool.guid: {'success': True, 'changes': True}}) self.assertListEqual(sorted(StorageRouterClient.node_config_recordings), ['1', '2']) expected_map = {'1': {'2': StorageDriver.DISTANCES.NEAR}, '2': {'1': StorageDriver.DISTANCES.NEAR}} configs = vpool.clusterregistry_client.get_node_configs() for config in configs: _validate_node_config(config, expected_map) # Running it again should not change anything StorageRouterClient.node_config_recordings = [] result = StorageDriverController.cluster_registry_checkup() self.assertDictEqual(result, {vpool.guid: {'success': True, 'changes': False}}) self.assertListEqual(sorted(StorageRouterClient.node_config_recordings), []) expected_map = {'1': {'2': StorageDriver.DISTANCES.NEAR}, '2': {'1': StorageDriver.DISTANCES.NEAR}} configs = vpool.clusterregistry_client.get_node_configs() for config in configs: _validate_node_config(config, expected_map) # Validate some error paths domain = structure['domains'][2] junction = structure['storagerouters'][1].domains[0] junction.domain = domain junction.save() vpool_config_path = 'file://opt/OpenvStorage/config/framework.json?key=/ovs/vpools/{0}/hosts/1/config'.format(vpool.guid) StorageRouterClient.exceptions['server_revision'] = {vpool_config_path: Exception('ClusterNotReachableException')} StorageRouterClient.node_config_recordings = [] result = StorageDriverController.cluster_registry_checkup() self.assertDictEqual(result, {vpool.guid: {'success': True, 'changes': True}}) self.assertListEqual(sorted(StorageRouterClient.node_config_recordings), ['2']) expected_map = {'1': {'2': StorageDriver.DISTANCES.INFINITE}, '2': {'1': StorageDriver.DISTANCES.INFINITE}} configs = vpool.clusterregistry_client.get_node_configs() for config in configs: _validate_node_config(config, expected_map)
def test_cluster_maintenance(self): """ Validates whether a cluster can be correctly created """ Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]}) Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]}) structure = Helper.build_service_structure({'storagerouters': [1, 2]}) storagerouters = structure['storagerouters'] System._machine_id = { storagerouters[1].ip: '1', storagerouters[2].ip: '2' } # Create new cluster mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir = mountpoint + '/test_create_cluster' info = ArakoonInstaller.create_cluster( 'test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir) reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': { 'tlogs': { 'dirs': {}, 'files': [] }, 'db': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '1', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[1].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY)) ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata']) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected) # Extending cluster mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir2 = mountpoint + '/test_extend_cluster' ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2) reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': { 'tlogs': { 'dirs': {}, 'files': [] }, 'db': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n{2}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '1,2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only' SSHClient._run_returns[catchup_command] = None SSHClient._run_recordings = [] ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False) self.assertIn(catchup_command, SSHClient._run_recordings) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected) # Shrinking cluster ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test') reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected)
def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_result): """ Add a service to the storage router :param service_storagerouter: Storage Router to add the service to :type service_storagerouter: StorageRouter :param arakoon_result: Port information :type arakoon_result: Dictionary :return: The newly created and added service """ new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = [ arakoon_result['client_port'], arakoon_result['messaging_port'] ] new_service.storagerouter = service_storagerouter new_service.save() return new_service cluster_name = 'voldrv' service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name('Arakoon') current_services = [] current_ips = [] for service in service_type.services: if service.name == service_name: current_services.append(service) current_ips.append(service.storagerouter.ip) all_sr_ips = [ storagerouter.ip for storagerouter in StorageRouterList.get_slaves() ] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition( storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len( current_services) == 0 and len(available_storagerouters) > 0: storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name=cluster_name, ip=storagerouter.ip, base_dir=partition.folder) current_services.append(add_service(storagerouter, result)) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) current_ips.append(storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver() if 0 < len(current_services) < len(available_storagerouters): for storagerouter, partition in available_storagerouters.iteritems( ): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster( current_services[0].storagerouter.ip, storagerouter.ip, cluster_name, partition.folder) add_service(storagerouter, result) current_ips.append(storagerouter.ip) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver()