def list_asds(mountpoint): """ Lists all ASDs found on a given mountpoint :param mountpoint: Mountpoint to list ASDs on :type mountpoint: str :return: Dictionary of ASDs :rtype: dict """ asds = {} try: for asd_id in os.listdir(mountpoint): if os.path.isdir('/'.join([mountpoint, asd_id])) and Configuration.exists(ASDController.ASD_CONFIG.format(asd_id)): asds[asd_id] = Configuration.get(ASDController.ASD_CONFIG.format(asd_id)) output, error = ASDController._local_client.run(['ls', '{0}/{1}/'.format(mountpoint, asd_id)], allow_nonzero=True, return_stderr=True) output += error if 'Input/output error' in output: asds[asd_id].update({'state': 'error', 'state_detail': 'io_error'}) continue service_name = ASDController.ASD_SERVICE_PREFIX.format(asd_id) if ServiceManager.has_service(service_name, ASDController._local_client): if ServiceManager.get_service_status(service_name, ASDController._local_client)[0] is False: asds[asd_id].update({'state': 'error', 'state_detail': 'service_failure'}) else: asds[asd_id].update({'state': 'ok'}) else: asds[asd_id].update({'state': 'error', 'state_detail': 'service_failure'}) except OSError as ex: ASDController._logger.info('Error collecting ASD information: {0}'.format(str(ex))) return asds
def add_maintenance_service(name, backend_guid, abm_name): """ Add a maintenance service with a specific name :param name: Name of the service to add :type name: str :param backend_guid: Backend for which the maintenance service needs to run :type backend_guid: str :param abm_name: Name of the ABM cluster :type abm_name: str """ if ServiceManager.has_service(name, MaintenanceController._local_client) is False: config_location = '/ovs/alba/backends/{0}/maintenance/config'.format(backend_guid) alba_config = Configuration.get_configuration_path(config_location) node_id = os.environ.get('ASD_NODE_ID') params = {'ALBA_CONFIG': alba_config, 'LOG_SINK': LogHandler.get_sink_path('alba_maintenance')} Configuration.set(config_location, json.dumps({ 'log_level': 'info', 'albamgr_cfg_url': Configuration.get_configuration_path('/ovs/arakoon/{0}/config'.format(abm_name)), 'read_preference': [] if node_id is None else [node_id] }, indent=4), raw=True) ServiceManager.add_service(name=MaintenanceController.MAINTENANCE_PREFIX, client=MaintenanceController._local_client, params=params, target_name=name) ServiceManager.start_service(name, MaintenanceController._local_client)
def _authorized(): """ Indicates whether a call is authenticated """ username = Configuration.get("/ovs/alba/asdnodes/{0}/config/main|username".format(NODE_ID)) password = Configuration.get("/ovs/alba/asdnodes/{0}/config/main|password".format(NODE_ID)) auth = request.authorization return auth and auth.username == username and auth.password == password
def write_config(self, ip=None): """ Writes the configuration down to in the format expected by Arakoon """ contents = self.export_ini() if self.filesystem is False: Configuration.set(self.config_path, contents, raw=True) else: client = self._load_client(ip) client.file_write(self.config_path, contents)
def delete_config(self, ip=None): """ Deletes a configuration file """ if self.filesystem is False: key = self.config_path if Configuration.exists(key, raw=True): Configuration.delete(key, raw=True) else: client = self._load_client(ip) client.file_delete(self.config_path)
def unregister_service(node_name, service_name): """ Un-register the metadata of a service from the configuration management :param node_name: Unused :type node_name: str :param service_name: Name of the service to clean from the configuration management :type service_name: str :return: None """ _ = node_name with open(Toolbox.BOOTSTRAP_FILE, 'r') as bs_file: node_id = json.load(bs_file)['node_id'] Configuration.delete(key='/ovs/alba/asdnodes/{0}/services/{1}'.format(node_id, Toolbox.remove_prefix(service_name, 'ovs-')))
def register_service(node_name, service_metadata): """ Register the metadata of the service to the configuration management :param node_name: Unused :type node_name: str :param service_metadata: Metadata of the service :type service_metadata: dict :return: None """ _ = node_name service_name = service_metadata['SERVICE_NAME'] with open(Toolbox.BOOTSTRAP_FILE, 'r') as bs_file: node_id = json.load(bs_file)['node_id'] Configuration.set(key='/ovs/alba/asdnodes/{0}/services/{1}'.format(node_id, Toolbox.remove_prefix(service_name, 'ovs-')), value=service_metadata)
def load_config(self, ip=None): """ Reads a configuration from reality """ if self.filesystem is False: contents = Configuration.get(self.config_path, raw=True) else: client = self._load_client(ip) contents = client.file_read(self.config_path) parser = RawConfigParser() parser.readfp(StringIO(contents)) self.nodes = [] self._extra_globals = {} for key in parser.options('global'): if key == 'plugins': self._plugins = [plugin.strip() for plugin in parser.get('global', 'plugins').split(',')] elif key == 'cluster_id': self.cluster_id = parser.get('global', 'cluster_id') elif key == 'cluster': pass # Ignore these else: self._extra_globals[key] = parser.get('global', key) for node in parser.get('global', 'cluster').split(','): node = node.strip() self.nodes.append(ArakoonNodeConfig(name=node, ip=parser.get(node, 'ip'), client_port=parser.get(node, 'client_port'), messaging_port=parser.get(node, 'messaging_port'), log_sinks=parser.get(node, 'log_sinks'), crash_log_sinks=parser.get(node, 'crash_log_sinks'), home=parser.get(node, 'home'), tlog_dir=parser.get(node, 'tlog_dir')))
def remove_asd(asd_id, mountpoint): """ Removes an ASD :param asd_id: ASD identifier :type asd_id: str :param mountpoint: Mountpoint of the ASDs disk :type mountpoint: str :return: None """ service_name = ASDController.ASD_SERVICE_PREFIX.format(asd_id) if ServiceManager.has_service(service_name, ASDController._local_client): ServiceManager.stop_service(service_name, ASDController._local_client) ServiceManager.remove_service(service_name, ASDController._local_client) try: ASDController._local_client.dir_delete('{0}/{1}'.format(mountpoint, asd_id)) except Exception as ex: ASDController._logger.warning('Could not clean ASD data: {0}'.format(ex)) Configuration.delete(ASDController.ASD_CONFIG_ROOT.format(asd_id), raw=True)
def services_running(self, target): """ Check all services are running :param target: Target to check :return: Boolean """ try: if target == 'config': self.log_message(target, 'Testing configuration store...', 0) from source.tools.configuration.configuration import Configuration try: Configuration.list('/') except Exception as ex: self.log_message(target, ' Error during configuration store test: {0}'.format(ex), 2) return False if Configuration.get_store() == 'arakoon': from source.tools.configuration.arakoon_config import ArakoonConfiguration from source.tools.pyrakoon.pyrakoon.compat import NoGuarantee client = ArakoonConfiguration.get_client() contents = client.get(Watcher.INTERNAL_CONFIG_KEY, consistency=NoGuarantee()) if Watcher.LOG_CONTENTS != contents: try: # Validate whether the contents are not corrupt parser = RawConfigParser() parser.readfp(StringIO(contents)) except Exception as ex: self.log_message(target, ' Configuration stored in configuration store seems to be corrupt: {0}'.format(ex), 2) return False temp_filename = '{0}~'.format(ArakoonConfiguration.CACC_LOCATION) with open(temp_filename, 'w') as config_file: config_file.write(contents) config_file.flush() os.fsync(config_file) os.rename(temp_filename, ArakoonConfiguration.CACC_LOCATION) if Watcher.LOG_CONTENTS is not None: self.log_message(target, ' Configuration changed, trigger restart', 1) sys.exit(1) Watcher.LOG_CONTENTS = contents self.log_message(target, ' Configuration store OK', 0) return True except Exception as ex: self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2) return False
def load_target_definition(source, allow_override=False): logging_target = {'type': 'console'} try: from source.tools.configuration.configuration import Configuration logging_target = Configuration.get('/ovs/framework/logging') except: pass target_type = logging_target.get('type', 'console') if allow_override is True and 'OVS_LOGTYPE_OVERRIDE' in os.environ: target_type = os.environ['OVS_LOGTYPE_OVERRIDE'] if target_type == 'redis': queue = logging_target.get('queue', '/ovs/logging') if '{0}' in queue: queue = queue.format(source) return {'type': 'redis', 'queue': '/{0}'.format(queue.lstrip('/')), 'host': logging_target.get('host', 'localhost'), 'port': logging_target.get('port', 6379)} if target_type == 'file': return {'type': 'file', 'filename': LogHandler.load_path(source)} return {'type': 'console'}
def setup(): """ Interactive setup part for initial asd manager configuration """ print Interactive.boxed_message(['ASD Manager setup']) print '- Verifying distribution' local_client = LocalClient() if ServiceManager.has_service(MANAGER_SERVICE, local_client): print '' # Spacing print Interactive.boxed_message(['The ASD Manager is already installed.']) sys.exit(1) ipaddresses = check_output("ip a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1", shell=True).strip().splitlines() ipaddresses = [found_ip.strip() for found_ip in ipaddresses if found_ip.strip() != '127.0.0.1'] if not ipaddresses: print Interactive.boxed_message(['Could not retrieve IP information on current node']) sys.exit(1) config = _validate_and_retrieve_pre_config() if config is None: api_ip = Interactive.ask_choice(ipaddresses, 'Select the public IP address to be used for the API') asd_ips = [] add_ips = True api_port = Interactive.ask_integer("Select the port to be used for the API", 1025, 65535, 8500) ipaddresses.append('All') while add_ips: current_ips = ' - Current selected IPs: {0}'.format(asd_ips) new_asd_ip = Interactive.ask_choice(ipaddresses, 'Select an IP address or all IP addresses to be used for the ASDs{0}'.format(current_ips if len(asd_ips) > 0 else ''), default_value='All') if new_asd_ip == 'All': ipaddresses.remove('All') asd_ips = [] add_ips = False else: asd_ips.append(new_asd_ip) ipaddresses.remove(new_asd_ip) add_ips = Interactive.ask_yesno("Do you want to add another IP?") asd_start_port = Interactive.ask_integer("Select the port to be used for the ASDs", 1025, 65435, 8600) store = Interactive.ask_choice(['Arakoon', 'Etcd'], question='Select the configuration management system', default_value='Arakoon').lower() else: store = config.get('store', 'arakoon') api_ip = config['api_ip'] api_port = config.get('api_port', 8500) asd_ips = config.get('asd_ips', []) asd_start_port = config.get('asd_start_port', 8600) if api_ip not in ipaddresses: print Interactive.boxed_message(['Unknown API IP provided, please choose from: {0}'.format(', '.join(ipaddresses))]) sys.exit(1) if set(asd_ips).difference(set(ipaddresses)): print Interactive.boxed_message(['Unknown ASD IP provided, please choose from: {0}'.format(', '.join(ipaddresses))]) sys.exit(1) if api_port in range(asd_start_port, asd_start_port + 100): print Interactive.boxed_message(['API port cannot be in the range of the ASD port + 100']) sys.exit(1) # Make sure to always have the information stored with open(PRECONFIG_FILE, 'w') as preconfig: preconfig.write(json.dumps({'asdmanager': {'store': store, 'api_ip': api_ip, 'asd_ips': asd_ips, 'api_port': api_port, 'asd_start_port': asd_start_port}}, indent=4)) if store == 'arakoon': from source.tools.configuration.arakoon_config import ArakoonConfiguration file_location = ArakoonConfiguration.CACC_LOCATION source_location = ArakoonConfiguration.CACC_SOURCE if not local_client.file_exists(file_location) and local_client.file_exists(source_location): # Try to copy automatically try: local_client.file_upload(file_location, source_location) except Exception: pass while not local_client.file_exists(file_location): print 'Please place a copy of the Arakoon\'s client configuration file at: {0}'.format(file_location) Interactive.ask_continue() bootstrap_location = Configuration.BOOTSTRAP_CONFIG_LOCATION if not local_client.file_exists(bootstrap_location): local_client.file_create(bootstrap_location) local_client.file_write(bootstrap_location, json.dumps({'configuration_store': store}, indent=4)) try: alba_node_id = Configuration.initialize(api_ip, api_port, asd_ips, asd_start_port) except: print '' if store == 'arakoon': print Interactive.boxed_message(['Could not connect to Arakoon']) else: print Interactive.boxed_message(['Could not connect to Etcd.', 'Please make sure an Etcd proxy is available, pointing towards an OpenvStorage cluster.']) sys.exit(1) with open(Toolbox.BOOTSTRAP_FILE, 'w') as bs_file: json.dump({'node_id': alba_node_id}, bs_file) ServiceManager.add_service(MANAGER_SERVICE, local_client) ServiceManager.add_service(WATCHER_SERVICE, local_client) print '- Starting watcher service' try: ServiceManager.start_service(WATCHER_SERVICE, local_client) except Exception as ex: Configuration.uninitialize(alba_node_id) print Interactive.boxed_message(['Starting watcher failed with error:', str(ex)]) sys.exit(1) print Interactive.boxed_message(['ASD Manager setup completed'])
def remove(silent=None): """ Interactive removal part for the ASD manager :param silent: If silent == '--force-yes' no question will be asked to confirm the removal :type silent: str :return: None """ os.environ['OVS_LOGTYPE_OVERRIDE'] = 'file' print '\n' + Interactive.boxed_message(['ASD Manager removal']) ############## # VALIDATION # ############## local_client = LocalClient() if not local_client.file_exists(filename=Toolbox.BOOTSTRAP_FILE): print '\n' + Interactive.boxed_message(['The ASD Manager has already been removed']) sys.exit(1) print ' - Validating configuration file' config = _validate_and_retrieve_pre_config() if config is None or 'store' not in config: print '\n' + Interactive.boxed_message(['Cannot remove the ASD manager because not all information could be retrieved from the pre-configuration file']) sys.exit(1) print ' - Validating node ID' with open(Toolbox.BOOTSTRAP_FILE, 'r') as bs_file: try: alba_node_id = json.loads(bs_file.read())['node_id'] except: print '\n' + Interactive.boxed_message(['JSON contents could not be retrieved from file {0}'.format(Toolbox.BOOTSTRAP_FILE)]) sys.exit(1) print ' - Validating configuration management' store = config['store'] try: Configuration.list(key='ovs') except: if store == 'arakoon': print '\n' + Interactive.boxed_message(['Could not connect to Arakoon']) else: print '\n' + Interactive.boxed_message(['Could not connect to Etcd.', 'Please make sure an Etcd proxy is available, pointing towards an OpenvStorage cluster.']) sys.exit(1) ################ # CONFIRMATION # ################ os.environ['ASD_NODE_ID'] = alba_node_id from source.app.api import API print ' - Retrieving ASD information' all_asds = {} try: all_asds = API.list_asds.original() except: print ' - ERROR: Failed to retrieve the ASD information' interactive = silent != '--force-yes' if interactive is True: message = 'Are you sure you want to continue?' if len(all_asds) > 0: print '\n\n+++ ALERT +++\n' message = 'DATA LOSS possible if proceeding! Continue?' proceed = Interactive.ask_yesno(message=message, default_value=False) if proceed is False: print '\n' + Interactive.boxed_message(['Abort removal']) sys.exit(1) ########### # REMOVAL # ########### print ' - Removing from configuration management' Configuration.uninitialize(node_id=alba_node_id) if len(all_asds) > 0: print ' - Removing disks' for device_id, disk_info in API.list_disks.original().iteritems(): if disk_info['available'] is True: continue try: print ' - Retrieving ASD information for disk {0}'.format(disk_info['device']) for asd_id, asd_info in API.list_asds_disk.original(disk_id=device_id).iteritems(): print ' - Removing ASD {0}'.format(asd_id) API.asd_delete.original(disk_id=device_id, asd_id=asd_id) API.delete_disk.original(disk_id=device_id) except Exception as ex: print ' - Deleting ASDs failed: {0}'.format(ex) print ' - Removing services' for service_name in API.list_maintenance_services.original()['services']: print ' - Removing service {0}'.format(service_name) API.remove_maintenance_service.original(name=service_name) for service_name in [WATCHER_SERVICE, MANAGER_SERVICE]: if ServiceManager.has_service(name=service_name, client=local_client): print ' - Removing service {0}'.format(service_name) ServiceManager.stop_service(name=service_name, client=local_client) ServiceManager.remove_service(name=service_name, client=local_client) if store == 'arakoon': from source.tools.configuration.arakoon_config import ArakoonConfiguration local_client.file_delete(filenames=ArakoonConfiguration.CACC_LOCATION) local_client.file_delete(filenames=Toolbox.BOOTSTRAP_FILE) print '\n' + Interactive.boxed_message(['ASD Manager removal completed'])
'asd_ips': (list, Toolbox.regex_ip, False), 'api_port': (int, {'min': 1025, 'max': 65535}, False), 'asd_start_port': (int, {'min': 1025, 'max': 65435}, False)}) except RuntimeError as rte: print Interactive.boxed_message(['The asd-manager pre-configuration file does not contain correct information\n{0}'.format(rte)]) sys.exit(1) return config if __name__ == '__main__': with open(Toolbox.BOOTSTRAP_FILE, 'r') as bootstrap_file: node_id = json.load(bootstrap_file)['node_id'] os.environ['ASD_NODE_ID'] = node_id try: asd_manager_config = Configuration.get('/ovs/alba/asdnodes/{0}/config/main'.format(node_id)) except: raise RuntimeError('Configuration management unavailable') if 'ip' not in asd_manager_config or 'port' not in asd_manager_config: raise RuntimeError('IP and/or port not available in configuration for ALBA node {0}'.format(node_id)) from source.app import app @app.before_first_request def setup_logging(): """ Configure logging :return: None """ if app.debug is False:
with open(Toolbox.BOOTSTRAP_FILE, 'r') as bootstrap_file: NODE_ID = json.load(bootstrap_file)['node_id'] os.environ['ASD_NODE_ID'] = NODE_ID CONFIG_ROOT = '/ovs/alba/asdnodes/{0}/config'.format(NODE_ID) CURRENT_VERSION = 1 _logger = LogHandler.get('asd-manager', name='post-update') _logger.info('Executing post-update logic of package openvstorage-sdm') with file_mutex('package_update_pu'): client = LocalClient('127.0.0.1', username='******') key = '{0}/versions'.format(CONFIG_ROOT) version = Configuration.get(key) if Configuration.exists(key) else 0 service_name = 'asd-manager' if ServiceManager.has_service(service_name, client) and ServiceManager.get_service_status(service_name, client)[0] is True: _logger.info('Stopping asd-manager service') ServiceManager.stop_service(service_name, client) if version < CURRENT_VERSION: try: # Put migration code here pass except: pass Configuration.set(key, CURRENT_VERSION) if ServiceManager.has_service(service_name, client) and ServiceManager.get_service_status(service_name, client)[0] is False:
def create_asd(partition_alias): """ Creates and starts an ASD on a given disk :param partition_alias: Alias of the partition of a disk (eg: /dev/disk/by-id/scsi-1ATA_TOSHIBA_MK2002TSKB_92M1KDMHF-part1) :type partition_alias: str :return: None """ all_asds = {} mountpoint = None for alias, mtpt in FSTab.read().iteritems(): all_asds.update(ASDController.list_asds(mtpt)) if alias == partition_alias: mountpoint = mtpt if mountpoint is None: raise RuntimeError('Failed to retrieve the mountpoint for partition with alias: {0}'.format(partition_alias)) # Fetch disk information disk_size = int(ASDController._local_client.run(['df', '-B', '1', '--output=size', mountpoint]).splitlines()[1]) # Find out appropriate disk size asds = 1.0 for asd_id in os.listdir(mountpoint): if os.path.isdir('/'.join([mountpoint, asd_id])) and Configuration.exists(ASDController.ASD_CONFIG.format(asd_id)): asds += 1 asd_size = int(math.floor(disk_size / asds)) for asd_id in os.listdir(mountpoint): if os.path.isdir('/'.join([mountpoint, asd_id])) and Configuration.exists(ASDController.ASD_CONFIG.format(asd_id)): config = json.loads(Configuration.get(ASDController.ASD_CONFIG.format(asd_id), raw=True)) config['capacity'] = asd_size config['rocksdb_block_cache_size'] = int(asd_size / 1024 / 4) Configuration.set(ASDController.ASD_CONFIG.format(asd_id), json.dumps(config, indent=4), raw=True) try: ServiceManager.send_signal(ASDController.ASD_SERVICE_PREFIX.format(asd_id), signal.SIGUSR1, ASDController._local_client) except Exception as ex: ASDController._logger.info('Could not send signal to ASD for reloading the quota: {0}'.format(ex)) # Prepare & start service asd_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(32)) ASDController._logger.info('Setting up service for disk {0}'.format(partition_alias)) homedir = '{0}/{1}'.format(mountpoint, asd_id) base_port = Configuration.get('{0}/network|port'.format(ASDController.CONFIG_ROOT)) ips = Configuration.get('{0}/network|ips'.format(ASDController.CONFIG_ROOT)) used_ports = [] for asd in all_asds.itervalues(): used_ports.append(asd['port']) if 'rora_port' in asd: used_ports.append(asd['rora_port']) asd_port = base_port rora_port = base_port + 1 while asd_port in used_ports: asd_port += 1 used_ports.append(asd_port) while rora_port in used_ports: rora_port += 1 asd_config = {'home': homedir, 'node_id': ASDController.NODE_ID, 'asd_id': asd_id, 'capacity': asd_size, 'log_level': 'info', 'port': asd_port, 'transport': 'tcp', 'rocksdb_block_cache_size': int(asd_size / 1024 / 4)} if Configuration.get('/ovs/framework/rdma'): asd_config['rora_port'] = rora_port asd_config['rora_transport'] = 'rdma' if ips is not None and len(ips) > 0: asd_config['ips'] = ips if Configuration.exists('{0}/extra'.format(ASDController.CONFIG_ROOT)): data = Configuration.get('{0}/extra'.format(ASDController.CONFIG_ROOT)) asd_config.update(data) Configuration.set(ASDController.ASD_CONFIG.format(asd_id), json.dumps(asd_config, indent=4), raw=True) service_name = ASDController.ASD_SERVICE_PREFIX.format(asd_id) params = {'CONFIG_PATH': Configuration.get_configuration_path('/ovs/alba/asds/{0}/config'.format(asd_id)), 'SERVICE_NAME': service_name, 'LOG_SINK': LogHandler.get_sink_path('alba_asd')} os.mkdir(homedir) ASDController._local_client.run(['chown', '-R', 'alba:alba', homedir]) ServiceManager.add_service('alba-asd', ASDController._local_client, params, service_name) ASDController.start_asd(asd_id)
def set_net(): """ Set IP information """ API._logger.info('Setting network information') Configuration.set('{0}/network|ips'.format(API.CONFIG_ROOT), json.loads(request.form['ips']))