def update_asd(asd, update_data): """ Updates an ASD with the 'update_data' provided :param asd: ASD to update :type asd: source.dal.objects.asd.ASD :param update_data: Data to update :type update_data: dict :raises ValueError: - When ASD configuration key is not present - When an unsupported key is passed in via 'update_data' :return: None :rtype: NoneType """ key_map = {'ips': 'hosts'} if not Configuration.exists(asd.config_key): raise ValueError( 'Failed to the configuration at location {0}'.format( asd.config_key)) config = Configuration.get(asd.config_key) for key, value in update_data.iteritems(): if key not in key_map: # Only updating IPs is supported for now raise ValueError( 'Unsupported property provided: {0}. Only IPs can be updated for now' .format(key)) setattr(asd, key_map[key], value) config[key] = value asd.save() Configuration.set(key=asd.config_key, value=config)
def services_running(self, target): """ Check all services are running :param target: Target to check :return: Boolean """ try: if target == 'config': self.log_message(target, 'Testing configuration store...', 0) from ovs_extensions.db.arakoon.pyrakoon.pyrakoon.compat import NoGuarantee from source.tools.arakooninstaller import ArakoonInstaller, ArakoonClusterConfig from source.tools.configuration import Configuration try: Configuration.list('/') except Exception as ex: self.log_message( target, ' Error during configuration store test: {0}'.format( ex), 2) return False with open(CACC_LOCATION) as config_file: contents = config_file.read() config = ArakoonClusterConfig(cluster_id='cacc', load_config=False) config.read_config(contents=contents) client = ArakoonInstaller.build_client(config) contents = client.get(Watcher.INTERNAL_CONFIG_KEY, consistency=NoGuarantee()) if Watcher.LOG_CONTENTS != contents: try: # Validate whether the contents are not corrupt parser = RawConfigParser() parser.readfp(StringIO(contents)) except Exception as ex: self.log_message( target, ' Configuration stored in configuration store seems to be corrupt: {0}' .format(ex), 2) return False temp_filename = '{0}~'.format(CACC_LOCATION) with open(temp_filename, 'w') as config_file: config_file.write(contents) config_file.flush() os.fsync(config_file) os.rename(temp_filename, CACC_LOCATION) if Watcher.LOG_CONTENTS is not None: self.log_message( target, ' Configuration changed, trigger restart', 1) sys.exit(1) Watcher.LOG_CONTENTS = contents self.log_message(target, ' Configuration store OK', 0) return True except Exception as ex: self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2) return False
def add_maintenance_service(name, alba_backend_guid, abm_name, read_preferences=None): """ Add a maintenance service with a specific name :param name: Name of the maintenance service to add :type name: str :param alba_backend_guid: ALBA Backend GUID for which the maintenance service needs to run :type alba_backend_guid: str :param abm_name: Name of the ABM cluster :type abm_name: str :param read_preferences: List of ALBA Node IDs (LOCAL) or ALBA IDs of linked ALBA Backends (GLOBAL) for the maintenance services where they should prioritize the READ actions :type read_preferences: list[str] :return: None :rtype: NoneType """ if MaintenanceController._service_manager.has_service( name, MaintenanceController._local_client) is False: alba_pkg_name, alba_version_cmd = PackageFactory.get_package_and_version_cmd_for( component=PackageFactory.COMP_ALBA) config_location = '{0}/config'.format( MaintenanceController.MAINTENANCE_KEY.format( alba_backend_guid, name)) params = { 'LOG_SINK': Logger.get_sink_path('alba_maintenance'), 'ALBA_CONFIG': Configuration.get_configuration_path(config_location), 'ALBA_PKG_NAME': alba_pkg_name, 'ALBA_VERSION_CMD': alba_version_cmd } Configuration.set( key=config_location, value={ 'log_level': 'info', 'albamgr_cfg_url': Configuration.get_configuration_path( '/ovs/arakoon/{0}/config'.format(abm_name)), 'read_preference': [] if read_preferences is None else read_preferences, 'multicast_discover_osds': False }) MaintenanceController._service_manager.add_service( name=MaintenanceController.MAINTENANCE_PREFIX, client=MaintenanceController._local_client, params=params, target_name=name) MaintenanceController._service_manager.start_service( name, MaintenanceController._local_client)
def set_net(request_data): # type: (dict) -> None """ Set IP information :param request_data: Data about the request (given by the decorator) :type request_data: dict :return: None :rtype: NoneType """ node_id = SettingList.get_setting_by_code(code='node_id').value Configuration.set( '{0}|ips'.format(ASD_NODE_CONFIG_NETWORK_LOCATION.format(node_id)), request_data['ips'])
def get_logging_info(cls): """ Retrieve logging information from the Configuration management :return: Dictionary containing logging information :rtype: dict """ try: return Configuration.get('/ovs/alba/logging') except (IOError, NotFoundException): return {}
def get_package_info(cls): """ Retrieve the package information related to the ASD Manager This must return a dictionary with keys: 'names', 'edition', 'binaries', 'non_blocking', 'version_commands' and 'mutually_exclusive' Names: These are the names of the packages split up per component related to this repository (alba-asdmanager) * Alba * PKG_MGR_SDM --> The code itself to deploy and run the ASD Manager * PKG_ALBA(_EE) --> ASDs and maintenance agents rely on the ALBA Binary * PKG_OVS_EXTENSIONS --> Extensions code is used by the alba-asdmanager Edition: Used for different purposes Binaries: The names of the packages that come with a binary (also split up per component) Non Blocking: Packages which are potentially not yet available on all releases. These should be removed once every release contains these packages by default Version Commands: The commandos used to determine which binary version is currently active Mutually Exclusive: Packages which are not allowed to be installed depending on the edition. Eg: ALBA_EE cannot be installed on a 'community' edition :return: A dictionary containing information about the expected packages to be installed :rtype: dict """ edition = Configuration.get_edition() if edition == cls.EDITION_COMMUNITY: return { 'names': { cls.COMP_ALBA: {cls.PKG_MGR_SDM, cls.PKG_ALBA, cls.PKG_OVS_EXTENSIONS} }, 'edition': edition, 'binaries': { cls.COMP_ALBA: {cls.PKG_ALBA} }, 'non_blocking': {cls.PKG_OVS_EXTENSIONS}, 'version_commands': { cls.PKG_ALBA: cls.VERSION_CMD_ALBA }, 'mutually_exclusive': {cls.PKG_ALBA_EE} } elif edition == cls.EDITION_ENTERPRISE: return { 'names': { cls.COMP_ALBA: {cls.PKG_MGR_SDM, cls.PKG_ALBA_EE, cls.PKG_OVS_EXTENSIONS} }, 'edition': edition, 'binaries': { cls.COMP_ALBA: {cls.PKG_ALBA_EE} }, 'non_blocking': {cls.PKG_OVS_EXTENSIONS}, 'version_commands': { cls.PKG_ALBA_EE: cls.VERSION_CMD_ALBA }, 'mutually_exclusive': {cls.PKG_ALBA} } else: raise ValueError( 'Unsupported edition found: "{0}"'.format(edition))
def remove_asd(asd): """ Remove an ASD :param asd: ASD to remove :type asd: source.dal.objects.asd.ASD :return: None :rtype: NoneType """ if ASDController._service_manager.has_service( asd.service_name, ASDController._local_client): ASDController._service_manager.stop_service( asd.service_name, ASDController._local_client) ASDController._service_manager.remove_service( asd.service_name, ASDController._local_client) try: ASDController._local_client.dir_delete('{0}/{1}'.format( asd.disk.mountpoint, asd.asd_id)) except Exception: ASDController._logger.exception('Could not clean ASD data') Configuration.delete(asd.config_key) asd.delete()
def remove_maintenance_service(name, alba_backend_guid=None): """ Remove a maintenance service with a specific name :param name: Name of the service to remove :type name: str :param alba_backend_guid: ALBA Backend GUID for which the maintenance service needs to be removed Defaults to None for backwards compatibility :type alba_backend_guid: str :return: None :rtype: NoneType """ if MaintenanceController._service_manager.has_service( name, MaintenanceController._local_client): MaintenanceController._service_manager.stop_service( name, MaintenanceController._local_client) MaintenanceController._service_manager.remove_service( name, MaintenanceController._local_client) if alba_backend_guid is not None: key = MaintenanceController.MAINTENANCE_KEY.format( alba_backend_guid, name) if Configuration.dir_exists(key=key): Configuration.delete(key=key)
def export(self): """ Exports the ASD information to a dict structure :return: Representation of the ASD as dict :rtype: dict """ if not self.has_config: raise RuntimeError('No configuration found for ASD {0}'.format( self.asd_id)) data = Configuration.get(self.config_key) for prop in self._properties: if prop.name == 'hosts': data['ips'] = getattr(self, prop.name) else: data[prop.name] = getattr(self, prop.name) if self.disk.state == 'MISSING': data.update({'state': 'error', 'state_detail': 'missing'}) else: output, error = ASD._local_client.run( ['ls', '{0}/{1}/'.format(self.disk.mountpoint, self.folder)], allow_nonzero=True, return_stderr=True) output += error if 'Input/output error' in output: data.update({'state': 'error', 'state_detail': 'io_error'}) elif ASD._service_manager.has_service(self.service_name, ASD._local_client): service_state = ASD._service_manager.get_service_status( self.service_name, ASD._local_client) if service_state == 'activating': data.update({ 'state': 'warning', 'state_detail': 'service_activating' }) elif service_state == 'active': data.update({'state': 'ok', 'state_detail': None}) else: data.update({ 'state': 'error', 'state_detail': 'service_failure' }) else: data.update({ 'state': 'error', 'state_detail': 'service_failure' }) return data
def authorized(cls): """ Indicates whether a call is authenticated """ # For backwards compatibility we first try to retrieve the node ID by using the bootstrap file try: with open(BOOTSTRAP_FILE) as bstr_file: node_id = json.load(bstr_file)['node_id'] except: node_id = SettingList.get_setting_by_code(code='node_id').value node_config = Configuration.get( ASD_NODE_CONFIG_MAIN_LOCATION.format(node_id)) username = node_config['username'] password = node_config['password'] auth = request.authorization return auth and auth.username == username and auth.password == password
def sync_disks(): # type: () -> None """ Syncs the disks Changes made to this code should be reflected in the framework DiskController.sync_with_reality call. :return: None :rtype: NoneType """ node_id = SettingList.get_setting_by_code(code='node_id').value s3 = Configuration.get( ASD_NODE_CONFIG_MAIN_LOCATION_S3.format(node_id), default=False) disks, name_alias_mapping = DiskTools.model_devices(s3=s3) disks_by_name = dict((disk.name, disk) for disk in disks) alias_name_mapping = name_alias_mapping.reverse_mapping() # Specific for the asd-manager: handle unique constraint exception DiskController._prepare_for_name_switch(disks) # Sync the model for disk in DiskList.get_disks(): generic_disk_model = None # type: GenericDisk for alias in disk.aliases: # IBS wont have alias if alias in alias_name_mapping: name = alias_name_mapping[alias].replace('/dev/', '') if name in disks_by_name: generic_disk_model = disks_by_name.pop(name) break # Partitioned loop, nvme devices no longer show up in alias_name_mapping if generic_disk_model is None and disk.name in disks_by_name and ( disk.name.startswith(tuple(['fio', 'loop', 'nvme']))): generic_disk_model = disks_by_name.pop(disk.name) if not generic_disk_model: # Remove disk / partitions if not reported by 'lsblk' DiskController._remove_disk_model(disk) else: # Update existing disks and their partitions DiskController._sync_disk_with_model(disk, generic_disk_model) # Create all disks and their partitions not yet modeled for disk_name, generic_disk_model in disks_by_name.iteritems(): DiskController._model_disk(generic_disk_model)
def migrate(cls): # type: () -> None """ Execute the migration logic. :return: None :rtype: NoneType """ with file_mutex('package_update_pu'): local_client = SSHClient(endpoint='127.0.0.1', username='******') # Override the created openvstorage_sdm_id during package install, with currently available SDM ID if local_client.file_exists(BOOTSTRAP_FILE): with open(BOOTSTRAP_FILE) as bstr_file: node_id = json.load(bstr_file)['node_id'] local_client.file_write(filename='/etc/openvstorage_sdm_id', contents=node_id + '\n') else: with open('/etc/openvstorage_sdm_id', 'r') as id_file: node_id = id_file.read().strip() key = '{0}/versions'.format( ASD_NODE_CONFIG_LOCATION.format(node_id)) version = Configuration.get(key) if Configuration.exists( key) else 0 asd_manager_service_name = 'asd-manager' if cls.service_manager.has_service( asd_manager_service_name, local_client) and cls.service_manager.get_service_status( asd_manager_service_name, local_client) == 'active': cls.logger.info('Stopping asd-manager service') cls.service_manager.stop_service(asd_manager_service_name, local_client) # @TODO: Move these migrations to alba_node.client.update_execute_migration_code() if version < cls.CURRENT_VERSION: try: # DB migrations from source.controllers.asd import ASDController from source.controllers.disk import DiskController from source.dal.asdbase import ASDBase from source.dal.lists.asdlist import ASDList from source.dal.lists.disklist import DiskList from source.dal.objects.asd import ASD if not local_client.file_exists('{0}/main.db'.format( ASDBase.DATABASE_FOLDER)): local_client.dir_create([ASDBase.DATABASE_FOLDER]) asd_map = dict( (asd.asd_id, asd) for asd in ASDList.get_asds()) DiskController.sync_disks() for disk in DiskList.get_usable_disks(): if disk.state == 'MISSING' or disk.mountpoint is None: continue for asd_id in local_client.dir_list(disk.mountpoint): if asd_id in asd_map: asd = asd_map[asd_id] else: asd = ASD() asd.disk = disk asd.asd_id = asd_id asd.folder = asd_id if asd.has_config: if asd.port is None or asd.hosts is None: config = Configuration.get( key=asd.config_key) asd.port = config['port'] asd.hosts = config.get('ips', []) asd.save() # Adjustment of open file descriptors for ASD/maintenance services to 8192 asd_service_names = list(ASDController.list_asd_services()) maintenance_service_names = list( MaintenanceController.get_services()) for service_name in asd_service_names + maintenance_service_names: if cls.service_manager.has_service( name=service_name, client=local_client): if cls.service_manager.__class__ == Systemd: path = '/lib/systemd/system/{0}.service'.format( service_name) check = 'LimitNOFILE=8192' else: path = '/etc/init/{0}.conf'.format( service_name) check = 'limit nofile 8192 8192' restart_required = False if os.path.exists(path): with open(path, 'r') as system_file: if check not in system_file.read(): restart_required = True if restart_required is False: continue configuration_key = ServiceFactory.SERVICE_CONFIG_KEY.format( node_id, service_name) if Configuration.exists(configuration_key): # Rewrite the service file cls.service_manager.add_service( name=ASDController.ASD_PREFIX if service_name in asd_service_names else MaintenanceController.MAINTENANCE_PREFIX, client=local_client, params=Configuration.get( configuration_key), target_name=service_name) # Let the update know that the ASD / maintenance services need to be restarted # Inside `if Configuration.exists`, because useless to rapport restart if we haven't rewritten service file ExtensionsToolbox.edit_version_file( client=local_client, package_name='alba', old_run_file='{0}/{1}.version'.format( ServiceFactory.RUN_FILE_DIR, service_name)) if cls.service_manager.__class__ == Systemd: local_client.run(['systemctl', 'daemon-reload']) # Version 3: Addition of 'ExecReload' for ASD/maintenance SystemD services if cls.service_manager.__class__ == Systemd: # Upstart does not have functionality to reload a process' configuration reload_daemon = False asd_service_names = list( ASDController.list_asd_services()) maintenance_service_names = list( MaintenanceController.get_services()) for service_name in asd_service_names + maintenance_service_names: if not cls.service_manager.has_service( name=service_name, client=local_client): continue path = '/lib/systemd/system/{0}.service'.format( service_name) if os.path.exists(path): with open(path, 'r') as system_file: if 'ExecReload' not in system_file.read(): reload_daemon = True configuration_key = ServiceFactory.SERVICE_CONFIG_KEY.format( node_id, service_name) if Configuration.exists( configuration_key): # No need to edit the service version file, since this change only requires a daemon-reload cls.service_manager.add_service( name=ASDController.ASD_PREFIX if service_name in asd_service_names else MaintenanceController. MAINTENANCE_PREFIX, client=local_client, params=Configuration.get( configuration_key), target_name=service_name) if reload_daemon is True: local_client.run(['systemctl', 'daemon-reload']) # Version 6: Introduction of Active Drive all_local_ips = OSFactory.get_manager().get_ip_addresses( client=local_client) for asd in ASDList.get_asds(): if asd.has_config: asd_config = Configuration.get(asd.config_key) if 'multicast' not in asd_config: asd_config['multicast'] = None if 'ips' in asd_config: asd_ips = asd_config['ips'] or all_local_ips else: asd_ips = all_local_ips asd.hosts = asd_ips asd_config['ips'] = asd_ips Configuration.set(asd.config_key, asd_config) asd.save() # Version 7: Moving flask certificate files to config dir for file_name in [ 'passphrase', 'server.crt', 'server.csr', 'server.key' ]: if local_client.file_exists( '/opt/asd-manager/source/{0}'.format( file_name)): local_client.file_move( source_file_name='/opt/asd-manager/source/{0}'. format(file_name), destination_file_name= '/opt/asd-manager/config/{0}'.format( file_name)) except: cls.logger.exception( 'Error while executing post-update code on node {0}'. format(node_id)) Configuration.set(key, cls.CURRENT_VERSION) if cls.service_manager.has_service( asd_manager_service_name, local_client) and cls.service_manager.get_service_status( asd_manager_service_name, local_client) != 'active': cls.logger.info('Starting asd-manager service') cls.service_manager.start_service(asd_manager_service_name, local_client) cls.logger.info('Post-update logic executed')
def execute_migration_code(cls): # type: () -> None """ Run some migration code after an update has been done :return: None :rtype: NoneType """ cls._logger.info('Starting out of band migrations for SDM nodes') ########################### # Start crucial migration # ########################### # Removal of bootstrap file and store API IP, API port and node ID in SQLite DB try: if cls._local_client.file_exists(BOOTSTRAP_FILE): cls._logger.info('Bootstrap file still exists. Retrieving node ID') with open(BOOTSTRAP_FILE) as bstr_file: node_id = json.load(bstr_file)['node_id'] else: node_id = SettingList.get_setting_by_code(code='node_id').value except Exception: cls._logger.exception('Unable to determine the node ID, cannot migrate') raise try: api_settings_map = {'api_ip': 'ip', 'api_port': 'port'} # Map settings code to keys in the Config management required_settings = ['node_id', 'migration_version'] + api_settings_map.keys() for settings_code in required_settings: try: _ = SettingList.get_setting_by_code(settings_code) except ObjectNotFoundException: cls._logger.info('Missing required settings: {0}'.format(settings_code)) if settings_code == 'node_id': value = node_id elif settings_code in api_settings_map.keys(): # Information must be extracted from Configuration main_config = Configuration.get(ASD_NODE_CONFIG_MAIN_LOCATION.format(node_id)) value = main_config[api_settings_map[settings_code]] elif settings_code == 'migration_version': # Introduce version for ASD Manager migration code value = 0 else: raise NotImplementedError('No action implemented for setting {0}'.format(settings_code)) cls._logger.info('Modeling Setting with code {0} and value {1}'.format(settings_code, value)) setting = Setting() setting.code = settings_code setting.value = value setting.save() if cls._local_client.file_exists(BOOTSTRAP_FILE): cls._logger.info('Removing the bootstrap file') cls._local_client.file_delete(BOOTSTRAP_FILE) except Exception: cls._logger.exception('Error during migration of code settings. Unable to proceed') raise ############################### # Start non-crucial migration # ############################### errors = [] migration_setting = SettingList.get_setting_by_code(code='migration_version') # Add installed package_name in version files and additional string replacements in service files try: if migration_setting.value < 1: cls._logger.info('Adding additional information to service files') edition = Configuration.get_edition() if edition == PackageFactory.EDITION_ENTERPRISE: for version_file_name in cls._local_client.file_list(directory=ServiceFactory.RUN_FILE_DIR): version_file_path = '{0}/{1}'.format(ServiceFactory.RUN_FILE_DIR, version_file_name) contents = cls._local_client.file_read(filename=version_file_path) if '{0}='.format(PackageFactory.PKG_ALBA) in contents: contents = contents.replace(PackageFactory.PKG_ALBA, PackageFactory.PKG_ALBA_EE) cls._local_client.file_write(filename=version_file_path, contents=contents) node_id = SettingList.get_setting_by_code(code='node_id').value asd_services = list(ASDController.list_asd_services()) maint_services = list(MaintenanceController.get_services()) for service_name in asd_services + maint_services: config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(node_id, service_name) if Configuration.exists(key=config_key): config = Configuration.get(key=config_key) if 'RUN_FILE_DIR' in config: continue config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR config['ALBA_PKG_NAME'] = PackageFactory.PKG_ALBA_EE config['ALBA_VERSION_CMD'] = PackageFactory.VERSION_CMD_ALBA Configuration.set(key=config_key, value=config) cls._service_manager.regenerate_service(name=ASDController.ASD_PREFIX if service_name in asd_services else MaintenanceController.MAINTENANCE_PREFIX, client=cls._local_client, target_name=service_name) except Exception as ex: cls._logger.exception('Failed to regenerate the ASD and Maintenance services') errors.append(ex) try: if migration_setting.value < 2: if System.get_component_identifier() not in Configuration.get(Configuration.get_registration_key(), default=[]): Configuration.register_usage(System.get_component_identifier()) except Exception as ex: cls._logger.exception('Failed to register the asd-manager') errors.append(ex) if len(errors) == 0: cls._logger.info('No errors during non-crucial migration. Saving the migration setting') # Save migration settings when no errors occurred migration_setting = SettingList.get_setting_by_code(code='migration_version') migration_setting.value = 2 migration_setting.save() cls._logger.info('Finished out of band migrations for SDM nodes')
def create_asd(disk): """ Creates and starts an ASD on a given disk :param disk: Disk on which to create an ASD :type disk: source.dal.objects.disk.Disk :return: None :rtype: NoneType """ # Validations if disk.state == 'MISSING': raise RuntimeError( 'Cannot create an ASD on missing disk {0}'.format(disk.name)) _node_id = SettingList.get_setting_by_code(code='node_id').value ipaddresses = Configuration.get('{0}|ips'.format( ASD_NODE_CONFIG_NETWORK_LOCATION.format(_node_id))) if len(ipaddresses) == 0: ipaddresses = OSFactory.get_manager().get_ip_addresses( client=ASDController._local_client) if len(ipaddresses) == 0: raise RuntimeError('Could not find any IP on the local node') alba_pkg_name, alba_version_cmd = PackageFactory.get_package_and_version_cmd_for( component='alba' ) # Call here, because this potentially raises error, which should happen before actually making changes # Fetch disk information disk_size = int( ASDController._local_client.run( ['df', '-B', '1', '--output=size', disk.mountpoint], timeout=5).splitlines()[1]) # Find out appropriate disk size asd_size = int(math.floor(disk_size / (len(disk.asds) + 1))) for asd in disk.asds: if asd.has_config: config = Configuration.get(asd.config_key) config['capacity'] = asd_size cache_size = ASDController.calculate_rocksdb_cache_size( is_ssd=disk.is_ssd) if cache_size: config.update({'rocksdb_block_cache_size': cache_size}) Configuration.set(asd.config_key, config) try: ASDController._service_manager.send_signal( asd.service_name, signal.SIGUSR1, ASDController._local_client) except Exception as ex: ASDController._logger.info( 'Could not send signal to ASD for reloading the quota: {0}' .format(ex)) used_ports = [] for asd in ASDList.get_asds(): if asd.has_config: config = Configuration.get(asd.config_key) used_ports.append(config['port']) if 'rora_port' in config: used_ports.append(config['rora_port']) # Prepare & start service ASDController._logger.info('Setting up service for disk {0}'.format( disk.name)) asd_id = ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(32)) homedir = '{0}/{1}'.format(disk.mountpoint, asd_id) base_port = Configuration.get('{0}|port'.format( ASD_NODE_CONFIG_NETWORK_LOCATION.format(_node_id))) asd_port = base_port rora_port = base_port + 1 while asd_port in used_ports: asd_port += 1 used_ports.append(asd_port) while rora_port in used_ports: rora_port += 1 asd_config = { 'ips': ipaddresses, 'home': homedir, 'port': asd_port, 'asd_id': asd_id, 'node_id': _node_id, 'capacity': asd_size, 'multicast': None, 'transport': 'tcp', 'log_level': 'info' } cache_size = ASDController.calculate_rocksdb_cache_size( is_ssd=disk.is_ssd) if cache_size: asd_config.update({'rocksdb_block_cache_size': cache_size}) if Configuration.get('/ovs/framework/rdma'): asd_config['rora_port'] = rora_port asd_config['rora_transport'] = 'rdma' if Configuration.exists('{0}/extra'.format( ASD_NODE_CONFIG_LOCATION.format(_node_id))): data = Configuration.get('{0}/extra'.format( ASD_NODE_CONFIG_LOCATION.format(_node_id))) asd_config.update(data) asd = ASD() asd.disk = disk asd.port = asd_port asd.hosts = ipaddresses asd.asd_id = asd_id asd.folder = asd_id asd.save() Configuration.set(asd.config_key, asd_config) params = { 'LOG_SINK': Logger.get_sink_path('alba-asd_{0}'.format(asd_id)), 'CONFIG_PATH': Configuration.get_configuration_path(asd.config_key), 'SERVICE_NAME': asd.service_name, 'ALBA_PKG_NAME': alba_pkg_name, 'ALBA_VERSION_CMD': alba_version_cmd } os.mkdir(homedir) ASDController._local_client.run(['chown', '-R', 'alba:alba', homedir]) ASDController._service_manager.add_service( name=ASDController.ASD_PREFIX, client=ASDController._local_client, params=params, target_name=asd.service_name) ASDController.start_asd(asd)
#!/usr/bin/python2 # Copyright 2015 Open vStorage NV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from source.app import app from source.tools.configuration import Configuration if __name__ == '__main__': context = ('server.crt', 'server.key') config = Configuration() config.migrate() app.run(host='0.0.0.0', port=8500, ssl_context=context, threaded=True)
def _sync_disks(): from source.controllers.disk import DiskController while True: DiskController.sync_disks() time.sleep(60) try: node_id = SettingList.get_setting_by_code(code='node_id').value except: # For backwards compatibility # After update SettingList has not been populated yet and post-update script of package will restart asd-manager with open(BOOTSTRAP_FILE) as bstr_file: node_id = json.load(bstr_file)['node_id'] try: asd_manager_config = Configuration.get( ASD_NODE_CONFIG_MAIN_LOCATION.format(node_id)) except: raise RuntimeError('Configuration management unavailable') if 'ip' not in asd_manager_config or 'port' not in asd_manager_config: raise RuntimeError( 'IP and/or port not available in configuration for ALBA node {0}'. format(node_id)) from source.app import app @app.before_first_request def setup_logging(): """ Configure logging :return: None
def remove(silent=None): """ Interactive removal part for the ASD manager :param silent: If silent == '--force-yes' no question will be asked to confirm the removal :type silent: str :return: None :rtype: NoneType """ _print_and_log(message='\n' + Interactive.boxed_message(['ASD Manager removal'])) local_client = SSHClient(endpoint='127.0.0.1', username='******') if not local_client.file_exists( filename='{0}/main.db'.format(Setting.DATABASE_FOLDER)): _print_and_log(level='error', message='\n' + Interactive.boxed_message( ['The ASD Manager has already been removed'])) sys.exit(1) _print_and_log(message=' - Validating configuration management') try: Configuration.list(key='ovs') except: _print_and_log( level='exception', message='\n' + Interactive.boxed_message(['Could not connect to Arakoon'])) sys.exit(1) _print_and_log(message=' - Retrieving ASD information') all_asds = {} try: all_asds = ASDList.get_asds() except: _print_and_log(level='exception', message=' - Failed to retrieve the ASD information') interactive = silent != '--force-yes' if interactive is True: message = 'Are you sure you want to continue?' if len(all_asds) > 0: _print_and_log(message='\n\n+++ ALERT +++\n', level='warning') message = 'DATA LOSS possible if proceeding! Continue?' proceed = Interactive.ask_yesno(message=message, default_value=False) if proceed is False: _print_and_log(level='error', message='\n' + Interactive.boxed_message(['Abort removal'])) sys.exit(1) if len(all_asds) > 0: _print_and_log(message=' - Removing disks') for disk in DiskList.get_disks(): if disk.available is True: continue try: _print_and_log( message=' - Retrieving ASD information for disk {0}'. format(disk.name)) for asd in disk.asds: _print_and_log( message=' - Removing ASD {0}'.format(asd.name)) ASDController.remove_asd(asd) DiskController.clean_disk(disk) except Exception: _print_and_log(level='exception', message=' - Deleting ASDs failed') _print_and_log(message=' - Removing services') service_manager = ServiceFactory.get_manager() for service in MaintenanceController.get_services(): service_name = service _print_and_log( message=' - Removing service {0}'.format(service_name)) guid = None for alba_backend_guid in Configuration.list(key='/ovs/alba/backends'): for maintenance_service_name in Configuration.list( key='/ovs/alba/backends/{0}/maintenance/'.format( alba_backend_guid)): if maintenance_service_name == service_name: guid = alba_backend_guid break MaintenanceController.remove_maintenance_service( name=service_name, alba_backend_guid=guid) for service_name in [WATCHER_SERVICE, MANAGER_SERVICE]: if service_manager.has_service(name=service_name, client=local_client): _print_and_log( message=' - Removing service {0}'.format(service_name)) service_manager.stop_service(name=service_name, client=local_client) service_manager.remove_service(name=service_name, client=local_client) _print_and_log(message=' - Removing from configuration management') remaining_users = Configuration.uninitialize() if not remaining_users: local_client.file_delete(filenames=CACC_LOCATION) local_client.file_delete( filenames='{0}/main.db'.format(Setting.DATABASE_FOLDER)) _print_and_log( message='\n' + Interactive.boxed_message(['ASD Manager removal completed']))
def _has_config(self): return Configuration.exists(self.config_key)
def setup(): """ Interactive setup part for initial asd manager configuration """ _print_and_log(message=Interactive.boxed_message(['ASD Manager setup'])) # Gather information ipaddresses = OSFactory.get_manager().get_ip_addresses() if not ipaddresses: _print_and_log( level='error', message='\n' + Interactive.boxed_message( ['Could not retrieve IP information on local node'])) sys.exit(1) validation_ip_addresses = copy.deepcopy(ipaddresses) local_client = SSHClient(endpoint='127.0.0.1', username='******') service_manager = ServiceFactory.get_manager() if service_manager.has_service(MANAGER_SERVICE, local_client): _print_and_log(level='error', message='\n' + Interactive.boxed_message( ['The ASD Manager is already installed.'])) sys.exit(1) config = _validate_and_retrieve_pre_config() interactive = len(config) == 0 ipmi_info = {'ip': None, 'username': None, 'pwd': None} if interactive is False: api_ip = config['api_ip'] api_port = config.get('api_port', 8500) asd_ips = config.get('asd_ips', []) asd_start_port = config.get('asd_start_port', 8600) configuration_store = config.get('configuration_store', 'arakoon') ipmi_info = config.get('ipmi', ipmi_info) else: api_ip = Interactive.ask_choice( choice_options=ipaddresses, question='Select the public IP address to be used for the API', sort_choices=True) api_port = Interactive.ask_integer( question="Select the port to be used for the API", min_value=1025, max_value=65535, default_value=8500) asd_ips = [] add_ips = True ipaddresses.append('All') while add_ips: current_ips = ' - Current selected IPs: {0}'.format(asd_ips) new_asd_ip = Interactive.ask_choice( choice_options=ipaddresses, question= "Select an IP address to be used for the ASDs or 'All' (All current and future interfaces: 0.0.0.0){0}" .format(current_ips if len(asd_ips) > 0 else ''), default_value='All') if new_asd_ip == 'All': ipaddresses.remove('All') asd_ips = [ ] # Empty list maps to all IPs - checked when configuring ASDs add_ips = False else: asd_ips.append(new_asd_ip) ipaddresses.remove(new_asd_ip) add_ips = Interactive.ask_yesno( "Do you want to add another IP?") asd_start_port = Interactive.ask_integer( question="Select the port to be used for the ASDs", min_value=1025, max_value=65435, default_value=8600) configuration_store = 'arakoon' message = 'Do you want to set IPMI configuration keys?' proceed = Interactive.ask_yesno(message=message, default_value=False) if proceed is True: ipmi_info['ip'] = Interactive.ask_string( message='Enter the IPMI IP address', regex_info={'regex': ExtensionsToolbox.regex_ip}) ipmi_info['username'] = Interactive.ask_string( message='Enter the IPMI username') ipmi_info['pwd'] = Interactive.ask_password( message='Enter the IPMI password') if api_ip not in validation_ip_addresses: _print_and_log( level='error', message='\n' + Interactive.boxed_message(lines=[ 'Invalid API IP {0} specified. Please choose from:'.format( api_ip) ] + [' * {0}'.format(ip) for ip in ipaddresses])) sys.exit(1) different_ips = set(asd_ips).difference(set(validation_ip_addresses)) if different_ips: _print_and_log( level='error', message='\n' + Interactive.boxed_message(lines=[ 'Invalid ASD IPs {0} specified. Please choose from:'.format( asd_ips) ] + [' * {0}'.format(ip) for ip in ipaddresses])) sys.exit(1) if api_port in range(asd_start_port, asd_start_port + 100): _print_and_log( level='error', message='\n' + Interactive.boxed_message( ['API port cannot be in the range of the ASD port + 100'])) sys.exit(1) if interactive is True: while not local_client.file_exists(CACC_LOCATION): _print_and_log( level='warning', message= ' - Please place a copy of the Arakoon\'s client configuration file at: {0}' .format(CACC_LOCATION)) Interactive.ask_continue() local_client.file_write(filename=CONFIG_STORE_LOCATION, contents=json.dumps( {'configuration_store': configuration_store}, indent=4)) node_id = Configuration.initialize( config={ 'api_ip': api_ip, 'asd_ips': asd_ips, 'api_port': api_port, 'asd_start_port': asd_start_port, 'ipmi': ipmi_info }) # Model settings _print_and_log(message=' - Store settings in DB') for code, value in { 'api_ip': api_ip, 'api_port': api_port, 'configuration_store': configuration_store, 'node_id': node_id }.iteritems(): setting = Setting() setting.code = code setting.value = value setting.save() # Deploy/start services _print_and_log(message=' - Deploying and starting services') service_manager.add_service(name=MANAGER_SERVICE, client=local_client) service_manager.add_service(name=WATCHER_SERVICE, client=local_client) _print_and_log(message=' - Starting watcher service') try: service_manager.start_service(name=WATCHER_SERVICE, client=local_client) except Exception: Configuration.uninitialize() _print_and_log(level='exception', message='\n' + Interactive.boxed_message(['Starting watcher failed'])) sys.exit(1) _print_and_log(message='\n' + Interactive.boxed_message(['ASD Manager setup completed']))