def remove(license_guid): """ Removes a license """ clients = {} storagerouters = StorageRouterList.get_storagerouters() try: for storagerouter in storagerouters: clients[storagerouter] = SSHClient(storagerouter.ip) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') lic = License(license_guid) if lic.can_remove is True: remove_functions = Toolbox.fetch_hooks( 'license', '{0}.remove'.format(lic.component)) result = remove_functions[0](component=lic.component, data=lic.data, valid_until=lic.valid_until, signature=lic.signature) if result is True: lic.delete() license_contents = [] for lic in LicenseList.get_licenses(): license_contents.append(lic.hash) for storagerouter in storagerouters: client = clients[storagerouter] client.file_write( '/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents))) return result return None
def _configure_arakoon_to_volumedriver(offline_node_ips=None): print 'Update existing vPools' logger.info('Update existing vPools') if offline_node_ips is None: offline_node_ips = [] for storagerouter in StorageRouterList.get_storagerouters(): config = ArakoonClusterConfig('voldrv') config.load_config() arakoon_nodes = [] for node in config.nodes: arakoon_nodes.append({'host': node.ip, 'port': node.client_port, 'node_id': node.name}) with Remote(storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], 'ovs') as remote: configuration_dir = '{0}/storagedriver/storagedriver'.format(EtcdConfiguration.get('/ovs/framework/paths|cfgdir')) if not remote.os.path.exists(configuration_dir): remote.os.makedirs(configuration_dir) for json_file in remote.os.listdir(configuration_dir): vpool_name = json_file.replace('.json', '') if json_file.endswith('.json'): if remote.os.path.exists('{0}/{1}.cfg'.format(configuration_dir, vpool_name)): continue # There's also a .cfg file, so this is an alba_proxy configuration file storagedriver_config = remote.StorageDriverConfiguration('storagedriver', vpool_name) storagedriver_config.load() storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id='voldrv', vregistry_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon', dls_arakoon_cluster_id='voldrv', dls_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.save(reload_config=True)
def remove(license_guid): """ Removes a license """ clients = {} storagerouters = StorageRouterList.get_storagerouters() try: for storagerouter in storagerouters: clients[storagerouter] = SSHClient(storagerouter.ip) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') lic = License(license_guid) if lic.can_remove is True: remove_functions = Toolbox.fetch_hooks('license', '{0}.remove'.format(lic.component)) result = remove_functions[0](component=lic.component, data=lic.data, valid_until=lic.valid_until, signature=lic.signature) if result is True: lic.delete() license_contents = [] for lic in LicenseList.get_licenses(): license_contents.append(lic.hash) for storagerouter in storagerouters: client = clients[storagerouter] client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents))) return result return None
def getVPoolByIPandPort(ip, port): for storagerouter in StorageRouterList.get_storagerouters(): for storagedriver in storagerouter.storagedrivers: if storagedriver.ports[ "edge"] == port and storagedriver.storage_ip == ip: return storagedriver.vpool raise RuntimeError("Could not find vpool for {}:{}".format(ip, port))
def print_current_mds_layout(): """ Prints the current MDS layout """ output = ['', 'Open vStorage - MDS debug information', '=====================================', 'timestamp: {0}'.format(time.time()), ''] for storagerouter in StorageRouterList.get_storagerouters(): output.append('+ {0} ({1})'.format(storagerouter.name, storagerouter.ip)) vpools = set(sd.vpool for sd in storagerouter.storagedrivers) for vpool in vpools: output.append(' + {0}'.format(vpool.name)) for mds_service in vpool.mds_services: if mds_service.service.storagerouter_guid == storagerouter.guid: masters, slaves = 0, 0 for junction in mds_service.vdisks: if junction.is_master: masters += 1 else: slaves += 1 capacity = mds_service.capacity if capacity == -1: capacity = 'infinite' load, _ = MDSServiceController.get_mds_load(mds_service) if load == float('inf'): load = 'infinite' else: load = '{0}%'.format(round(load, 2)) output.append(' + {0} - port {1} - {2} master(s), {3} slave(s) - capacity: {4}, load: {5}'.format( mds_service.number, mds_service.service.ports[0], masters, slaves, capacity, load )) print '\n'.join(output)
def list(self): """ Overview of all StorageRouters :return: List of StorageRouters :rtype: list[ovs.dal.hybrids.storagerouter.StorageRouter] """ return StorageRouterList.get_storagerouters()
def _run_and_validate_dtl_checkup(self, vdisk, validations): """ Execute the DTL checkup for a vDisk and validate the settings afterwards """ single_node = len(StorageRouterList.get_storagerouters()) == 1 VDiskController.dtl_checkup(vdisk_guid=vdisk.guid) config = vdisk.storagedriver_client.get_dtl_config(vdisk.volume_id) config_mode = vdisk.storagedriver_client.get_dtl_config_mode(vdisk.volume_id) msg = '{0} node - {{0}} - Actual: {{1}} - Expected: {{2}}'.format('Single' if single_node is True else 'Multi') validations.append({'key': 'config_mode', 'value': DTLConfigMode.MANUAL}) for validation in validations: key = validation['key'] value = validation['value'] if key == 'config': actual_value = config elif key == 'host': actual_value = config.host elif key == 'port': actual_value = config.port elif key == 'mode': actual_value = config.mode else: actual_value = config_mode if isinstance(value, list): self.assertTrue(expr=actual_value in value, msg=msg.format(key.capitalize(), actual_value, ', '.join(value))) else: self.assertEqual(first=actual_value, second=value, msg=msg.format(key.capitalize(), actual_value, value)) return config
def pulse(): """ Update the heartbeats for the Current Routers :return: None """ logger = Logger('extensions-generic') machine_id = System.get_my_machine_id() current_time = int(time.time()) routers = StorageRouterList.get_storagerouters() for node in routers: if node.machine_id == machine_id: with volatile_mutex('storagerouter_heartbeat_{0}'.format( node.guid)): node_save = StorageRouter(node.guid) node_save.heartbeats['process'] = current_time node_save.save() StorageRouterController.ping.s( node.guid, current_time).apply_async( routing_key='sr.{0}'.format(machine_id)) else: try: # check timeout of other nodes and clear arp cache if node.heartbeats and 'process' in node.heartbeats: if current_time - node.heartbeats[ 'process'] >= HeartBeat.ARP_TIMEOUT: check_output("/usr/sbin/arp -d '{0}'".format( node.name.replace(r"'", r"'\''")), shell=True) except CalledProcessError: logger.exception('Error clearing ARP cache')
def pulse(): """ Update the heartbeats for the Current Routers :return: None """ logger = LogHandler.get('extensions', name='heartbeat') machine_id = System.get_my_machine_id() current_time = int(time.time()) routers = StorageRouterList.get_storagerouters() for node in routers: if node.machine_id == machine_id: with volatile_mutex('storagerouter_heartbeat_{0}'.format(node.guid)): node_save = StorageRouter(node.guid) node_save.heartbeats['process'] = current_time node_save.save() StorageRouterController.ping.s(node.guid, current_time).apply_async(routing_key='sr.{0}'.format(machine_id)) else: try: # check timeout of other nodes and clear arp cache if node.heartbeats and 'process' in node.heartbeats: if current_time - node.heartbeats['process'] >= HeartBeat.ARP_TIMEOUT: check_output("/usr/sbin/arp -d '{0}'".format(node.name.replace(r"'", r"'\''")), shell=True) except CalledProcessError: logger.exception('Error clearing ARP cache')
def _configure_arakoon_to_volumedriver(): print 'Update existing vPools' logger.info('Update existing vPools') for storagerouter in StorageRouterList.get_storagerouters(): with Remote(storagerouter.ip, [os, RawConfigParser, Configuration, StorageDriverConfiguration, ArakoonManagementEx], 'ovs') as remote: arakoon_cluster_config = remote.ArakoonManagementEx().getCluster('voldrv').getClientConfig() arakoon_nodes = [] for node_id, node_config in arakoon_cluster_config.iteritems(): arakoon_nodes.append({'host': node_config[0][0], 'port': node_config[1], 'node_id': node_id}) configuration_dir = '{0}/storagedriver/storagedriver'.format( remote.Configuration.get('ovs.core.cfgdir')) if not remote.os.path.exists(configuration_dir): remote.os.makedirs(configuration_dir) for json_file in remote.os.listdir(configuration_dir): vpool_name = json_file.replace('.json', '') if json_file.endswith('.json'): if remote.os.path.exists('{0}/{1}.cfg'.format(configuration_dir, vpool_name)): continue # There's also a .cfg file, so this is an alba_proxy configuration file storagedriver_config = remote.StorageDriverConfiguration('storagedriver', vpool_name) storagedriver_config.load() storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id='voldrv', vregistry_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon', dls_arakoon_cluster_id='voldrv', dls_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.save()
def _dtl_status(self): """ Retrieve the DTL status for a vDisk """ sd_status = self.info.get('failover_mode', 'UNKNOWN').lower() if sd_status == '': sd_status = 'unknown' if sd_status != 'ok_standalone': return sd_status # Verify whether 'ok_standalone' is the correct status for this vDisk vpool_dtl = self.vpool.configuration['dtl_enabled'] if self.has_manual_dtl is True or vpool_dtl is False: return sd_status domains = [] possible_dtl_targets = set() for sr in StorageRouterList.get_storagerouters(): if sr.guid == self.storagerouter_guid: domains = [junction.domain for junction in sr.domains] elif len(sr.storagedrivers) > 0: possible_dtl_targets.add(sr) if len(domains) > 0: possible_dtl_targets = set() for domain in domains: possible_dtl_targets.update( StorageRouterList.get_primary_storagerouters_for_domain( domain)) if len(possible_dtl_targets) == 0: return sd_status return 'checkup_required'
def _configure_arakoon_to_volumedriver(): print "Update existing vPools" logger.info("Update existing vPools") for storagerouter in StorageRouterList.get_storagerouters(): config = ArakoonClusterConfig("voldrv") config.load_config() arakoon_nodes = [] for node in config.nodes: arakoon_nodes.append({"host": node.ip, "port": node.client_port, "node_id": node.name}) with Remote( storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], "ovs" ) as remote: configuration_dir = "{0}/storagedriver/storagedriver".format( EtcdConfiguration.get("/ovs/framework/paths|cfgdir") ) if not remote.os.path.exists(configuration_dir): remote.os.makedirs(configuration_dir) for json_file in remote.os.listdir(configuration_dir): vpool_name = json_file.replace(".json", "") if json_file.endswith(".json"): if remote.os.path.exists("{0}/{1}.cfg".format(configuration_dir, vpool_name)): continue # There's also a .cfg file, so this is an alba_proxy configuration file storagedriver_config = remote.StorageDriverConfiguration("storagedriver", vpool_name) storagedriver_config.load() storagedriver_config.configure_volume_registry( vregistry_arakoon_cluster_id="voldrv", vregistry_arakoon_cluster_nodes=arakoon_nodes ) storagedriver_config.configure_distributed_lock_store( dls_type="Arakoon", dls_arakoon_cluster_id="voldrv", dls_arakoon_cluster_nodes=arakoon_nodes ) storagedriver_config.save(reload_config=True)
def on_demote(cluster_ip, master_ip): """ Handles the demote for the StorageDrivers :param cluster_ip: IP of the node to demote :param master_ip: IP of the master node """ client = SSHClient(cluster_ip, username='******') servicetype = ServiceTypeList.get_by_name('Arakoon') current_service = None remaining_ips = [] for service in servicetype.services: if service.name == 'arakoon-voldrv': if service.storagerouter.ip == cluster_ip: current_service = service else: remaining_ips.append(service.storagerouter.ip) if current_service is not None: print '* Shrink StorageDriver cluster' ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv') if ServiceManager.has_service(current_service.name, client=client) is True: ServiceManager.stop_service(current_service.name, client=client) ServiceManager.remove_service(current_service.name, client=client) ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips) current_service.delete() for storagerouter in StorageRouterList.get_storagerouters(): ArakoonInstaller.deploy_to_slave(master_ip, storagerouter.ip, 'voldrv') StorageDriverController._configure_arakoon_to_volumedriver()
def _dtl_status(self): """ Retrieve the DTL status for a vDisk """ sd_status = self.info.get('failover_mode', 'UNKNOWN').lower() if sd_status == '': sd_status = 'unknown' if sd_status != 'ok_standalone': return sd_status # Verify whether 'ok_standalone' is the correct status for this vDisk vpool_dtl = self.vpool.configuration['dtl_enabled'] if self.has_manual_dtl is True or vpool_dtl is False: return sd_status domains = [] possible_dtl_targets = set() for sr in StorageRouterList.get_storagerouters(): if sr.guid == self.storagerouter_guid: domains = [junction.domain for junction in sr.domains] elif len(sr.storagedrivers) > 0: possible_dtl_targets.add(sr) if len(domains) > 0: possible_dtl_targets = set() for domain in domains: possible_dtl_targets.update(StorageRouterList.get_primary_storagerouters_for_domain(domain)) if len(possible_dtl_targets) == 0: return sd_status return 'checkup_required'
def pulse(): """ Update the heartbeats for all Storage Routers :return: None """ logger = LogHandler.get('extensions', name='heartbeat') current_time = int(time.time()) machine_id = System.get_my_machine_id() amqp = '{0}://{1}:{2}@{3}//'.format(EtcdConfiguration.get('/ovs/framework/messagequeue|protocol'), EtcdConfiguration.get('/ovs/framework/messagequeue|user'), EtcdConfiguration.get('/ovs/framework/messagequeue|password'), EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(machine_id))) celery_path = OSManager.get_path('celery') worker_states = check_output("{0} inspect ping -b {1} --timeout=5 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true".format(celery_path, amqp), shell=True) routers = StorageRouterList.get_storagerouters() for node in routers: if node.heartbeats is None: node.heartbeats = {} if 'celery@{0}: OK'.format(node.name) in worker_states: node.heartbeats['celery'] = current_time if node.machine_id == machine_id: node.heartbeats['process'] = current_time else: try: # check timeout of other nodes and clear arp cache if node.heartbeats and 'process' in node.heartbeats: if current_time - node.heartbeats['process'] >= HeartBeat.ARP_TIMEOUT: check_output("/usr/sbin/arp -d {0}".format(node.name), shell=True) except CalledProcessError: logger.exception('Error clearing ARP cache') node.save()
def _run_and_validate_dtl_checkup(self, vdisk, validations): """ Execute the DTL checkup for a vDisk and validate the settings afterwards """ single_node = len(StorageRouterList.get_storagerouters()) == 1 VDiskController.dtl_checkup(vdisk_guid=vdisk.guid) config = vdisk.storagedriver_client.get_dtl_config(vdisk.volume_id) config_mode = vdisk.storagedriver_client.get_dtl_config_mode(vdisk.volume_id) msg = '{0} node - {{0}} - Actual: {{1}} - Expected: {{2}}'.format('Single' if single_node is True else 'Multi') validations.append({'key': 'config_mode', 'value': DTLConfigMode.MANUAL}) for validation in validations: key = validation['key'] value = validation['value'] if key == 'config': actual_value = config elif key == 'host': actual_value = config.host elif key == 'port': actual_value = config.port elif key == 'mode': actual_value = config.mode else: actual_value = config_mode if isinstance(value, list): self.assertTrue(expr=actual_value in value, msg=msg.format(key.capitalize(), actual_value, ', '.join(value))) else: self.assertEqual(first=actual_value, second=value, msg=msg.format(key.capitalize(), actual_value, value)) return config
def get_stats_storagerouters(cls): """ Retrieve amount of vDisks and some read/write statistics for all StorageRouters """ if cls._config is None: cls.validate_and_retrieve_config() stats = [] errors = False environment = cls._config['environment'] for storagerouter in StorageRouterList.get_storagerouters(): if len(storagerouter.storagedrivers) == 0: cls._logger.debug('StorageRouter {0} does not have any StorageDrivers linked to it, skipping'.format(storagerouter.name)) continue try: statistics = storagerouter.statistics stats.append({'tags': {'environment': environment, 'storagerouter_name': storagerouter.name}, 'fields': {'read_byte': statistics['data_read'], 'write_byte': statistics['data_written'], 'operations': statistics['4k_operations'], 'amount_vdisks': len(storagerouter.vdisks_guids), 'read_operations': statistics['4k_read_operations'], 'write_operations': statistics['4k_write_operations']}, 'measurement': 'storagerouter'}) except Exception: errors = True cls._logger.exception('Retrieving statistics for StorageRouter {0} failed'.format(storagerouter.name)) return errors, stats
def process_response(self, request, response): """ Processes responses """ _ = self # Timings if isinstance(response, OVSResponse): if hasattr(request, '_entry_time'): # noinspection PyProtectedMember response.timings['total'] = [ time.time() - request._entry_time, 'Total' ] response.build_timings() # Process CORS responses if 'HTTP_ORIGIN' in request.META: path = request.path storagerouters = StorageRouterList.get_storagerouters() allowed_origins = [ 'https://{0}'.format(storagerouter.ip) for storagerouter in storagerouters ] if request.META[ 'HTTP_ORIGIN'] in allowed_origins or '/swagger.json' in path: response['Access-Control-Allow-Origin'] = request.META[ 'HTTP_ORIGIN'] response[ 'Access-Control-Allow-Headers'] = 'x-requested-with, content-type, accept, origin, authorization' response[ 'Access-Control-Allow-Methods'] = 'GET, POST, PUT, PATCH, DELETE, OPTIONS' return response
def test_ssh_connectivity(): """ Validates whether all nodes can SSH into eachother """ MonitoringController._logger.info('Starting SSH connectivity test...') ips = [sr.ip for sr in StorageRouterList.get_storagerouters()] for ip in ips: for primary_username in ['root', 'ovs']: try: with remote(ip, [SSHClient], username=primary_username) as rem: for local_ip in ips: for username in ['root', 'ovs']: message = '* Connection from {0}@{1} to {2}@{3}... {{0}}'.format( primary_username, ip, username, local_ip) try: c = rem.SSHClient(local_ip, username=username) assert c.run(['whoami' ]).strip() == username message = message.format('OK') logger = MonitoringController._logger.info except Exception as ex: message = message.format(ex.message) logger = MonitoringController._logger.error logger(message) except Exception as ex: MonitoringController._logger.error( '* Could not connect to {0}@{1}: {2}'.format( primary_username, ip, ex.message)) MonitoringController._logger.info('Finished')
def get_storagerouters(): """ Fetch the storagerouters :return: list with storagerouters :rtype: list """ return StorageRouterList.get_storagerouters()
def get_available_actions(self): """ Gets a list of all available actions """ actions = [] storagerouters = StorageRouterList.get_storagerouters() if len(storagerouters) > 1: actions.append('MOVE_AWAY') return Response(actions, status=status.HTTP_200_OK)
def get_available_actions(self): """ Gets a list of all available actions """ actions = [] storagerouters = StorageRouterList.get_storagerouters() if len(storagerouters) > 1: actions.append('MOVE_AWAY') return Response(actions, status=status.HTTP_200_OK)
def get_available_actions(self): """ Gets a list of all available actions """ actions = [] storagerouters = StorageRouterList.get_storagerouters() if len(storagerouters) > 1: actions.append('MOVE_AWAY') return actions
def get_available_actions(self): """ Gets a list of all available actions """ actions = [] storagerouters = StorageRouterList.get_storagerouters() if len(storagerouters) > 1: actions.append('MOVE_AWAY') return actions
def apply(license_string): """ Applies a license. It will apply as much licenses as possible, however, it won't fail on invalid licenses as it will simply skip them. """ try: clients = {} storagerouters = StorageRouterList.get_storagerouters() try: for storagerouter in storagerouters: clients[storagerouter] = SSHClient(storagerouter.ip) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') data = LicenseController._decode(license_string) for component in data: cdata = data[component] name = cdata['name'] data = cdata['data'] token = cdata['token'] valid_until = float( cdata['valid_until']) if 'valid_until' in cdata else None if valid_until is not None and valid_until <= time.time(): continue signature = cdata['signature'] if 'signature' in cdata else None validate_functions = Toolbox.fetch_hooks( 'license', '{0}.validate'.format(component)) apply_functions = Toolbox.fetch_hooks( 'license', '{0}.apply'.format(component)) if len(validate_functions) == 1 and len(apply_functions) == 1: valid, metadata = validate_functions[0]( component=component, data=data, signature=signature) if valid is True: success = apply_functions[0](component=component, data=data, signature=signature) if success is True: license_object = LicenseList.get_by_component( component) if license_object is None: license_object = License() license_object.component = component license_object.name = name license_object.token = token license_object.data = data license_object.valid_until = valid_until license_object.signature = signature license_object.save() license_contents = [] for lic in LicenseList.get_licenses(): license_contents.append(lic.hash) for storagerouter in storagerouters: client = clients[storagerouter] client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents))) except Exception, ex: logger.exception('Error applying license: {0}'.format(ex)) return None
def list(self, query=None): """ Overview of all Storage Routers """ if query is None: return StorageRouterList.get_storagerouters() else: query = json.loads(query) return DataList(StorageRouter, query)
def list(self, query=None): """ Overview of all Storage Routers """ if query is None: return StorageRouterList.get_storagerouters() else: query = json.loads(query) return DataList(StorageRouter, query)
def list(self, query=None): """ Overview of all Storage Routers """ if query is None: return StorageRouterList.get_storagerouters() else: query = json.loads(query) query_result = DataList({"object": StorageRouter, "data": DataList.select.GUIDS, "query": query}).data return DataObjectList(query_result, StorageRouter)
def list(self, query=None): """ Overview of all Storage Routers :param query: A query to filter the StorageRouters :type query: DataQuery """ if query is None: return StorageRouterList.get_storagerouters() else: return DataList(StorageRouter, query)
def collapse_arakoon(): """ Collapse Arakoon's Tlogs :return: None """ from ovs_extensions.generic.toolbox import ExtensionsToolbox GenericController._logger.info('Arakoon collapse started') cluster_info = [] storagerouters = StorageRouterList.get_storagerouters() if os.environ.get('RUNNING_UNITTESTS') != 'True': cluster_info = [('cacc', storagerouters[0])] cluster_names = [] for service in ServiceList.get_services(): if service.is_internal is True and service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON, ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR): cluster = ExtensionsToolbox.remove_prefix(service.name, 'arakoon-') if cluster in cluster_names and cluster not in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST]: continue cluster_names.append(cluster) cluster_info.append((cluster, service.storagerouter)) workload = {} cluster_config_map = {} for cluster, storagerouter in cluster_info: GenericController._logger.debug(' Collecting info for cluster {0}'.format(cluster)) ip = storagerouter.ip if cluster in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST] else None try: config = ArakoonClusterConfig(cluster_id=cluster, source_ip=ip) cluster_config_map[cluster] = config except: GenericController._logger.exception(' Retrieving cluster information on {0} for {1} failed'.format(storagerouter.ip, cluster)) continue for node in config.nodes: if node.ip not in workload: workload[node.ip] = {'node_id': node.name, 'clusters': []} workload[node.ip]['clusters'].append((cluster, ip)) for storagerouter in storagerouters: try: if storagerouter.ip not in workload: continue node_workload = workload[storagerouter.ip] client = SSHClient(storagerouter) for cluster, ip in node_workload['clusters']: try: GenericController._logger.debug(' Collapsing cluster {0} on {1}'.format(cluster, storagerouter.ip)) client.run(['arakoon', '--collapse-local', node_workload['node_id'], '2', '-config', cluster_config_map[cluster].external_config_path]) GenericController._logger.debug(' Collapsing cluster {0} on {1} completed'.format(cluster, storagerouter.ip)) except: GenericController._logger.exception(' Collapsing cluster {0} on {1} failed'.format(cluster, storagerouter.ip)) except UnableToConnectException: GenericController._logger.error(' Could not collapse any cluster on {0} (not reachable)'.format(storagerouter.name)) GenericController._logger.info('Arakoon collapse finished')
def get(self, request, *args, **kwargs): """ Fetches metadata """ _ = args, kwargs data = {'authenticated': False, 'authentication_state': None, 'username': None, 'userguid': None, 'roles': [], 'storagerouter_ips': [sr.ip for sr in StorageRouterList.get_storagerouters()], 'versions': list(settings.VERSION), 'plugins': {}} try: # Gather plugin metadata plugins = {} # - Backends. BackendType plugins must set the has_plugin flag on True for backend_type in BackendTypeList.get_backend_types(): if backend_type.has_plugin is True: if backend_type.code not in plugins: plugins[backend_type.code] = [] plugins[backend_type.code] += ['backend', 'gui'] data['plugins'] = plugins # Gather authorization metadata if 'HTTP_AUTHORIZATION' not in request.META: return HttpResponse, dict(data.items() + {'authentication_state': 'unauthenticated'}.items()) authorization_type, access_token = request.META['HTTP_AUTHORIZATION'].split(' ') if authorization_type != 'Bearer': return HttpResponse, dict(data.items() + {'authentication_state': 'invalid_authorization_type'}.items()) tokens = BearerTokenList.get_by_access_token(access_token) if len(tokens) != 1: return HttpResponse, dict(data.items() + {'authentication_state': 'invalid_token'}.items()) token = tokens[0] if token.expiration < time.time(): for junction in token.roles.itersafe(): junction.delete() token.delete() return HttpResponse, dict(data.items() + {'authentication_state': 'token_expired'}.items()) # Gather user metadata user = token.client.user if not user.is_active: return HttpResponse, dict(data.items() + {'authentication_state': 'inactive_user'}.items()) roles = [j.role.code for j in token.roles] return HttpResponse, dict(data.items() + {'authenticated': True, 'authentication_state': 'authenticated', 'username': user.username, 'userguid': user.guid, 'roles': roles, 'plugins': plugins}.items()) except Exception as ex: logger.exception('Unexpected exception: {0}'.format(ex)) return HttpResponse, dict(data.items() + {'authentication_state': 'unexpected_exception'}.items())
def get_storagerouter_ips(): """ Fetch all the ip addresses in this cluster :return: list with storagerouter ips :rtype: list """ return [ storagerouter.ip for storagerouter in StorageRouterList.get_storagerouters() ]
def monitor_mds_layout(): """ Prints the current MDS layout :return: None :rtype: NoneType """ try: while True: output = [ '', 'Open vStorage - MDS debug information', '=====================================', 'timestamp: {0}'.format(datetime.datetime.now()), '' ] vpools_deployed = False for storagerouter in sorted( StorageRouterList.get_storagerouters(), key=lambda k: k.name): vpools = set(sd.vpool for sd in storagerouter.storagedrivers) if len(vpools) > 0: vpools_deployed = True output.append('+ {0} ({1})'.format( storagerouter.name, storagerouter.ip)) for vpool in sorted(vpools, key=lambda k: k.name): output.append(' + {0}'.format(vpool.name)) for mds_service in sorted(vpool.mds_services, key=lambda k: k.number): if mds_service.service.storagerouter_guid == storagerouter.guid: masters, slaves = 0, 0 for junction in mds_service.vdisks: if junction.is_master: masters += 1 else: slaves += 1 capacity = mds_service.capacity if capacity == -1: capacity = 'infinite' load, _ = MDSServiceController.get_mds_load( mds_service) if load == float('inf'): load = 'infinite' else: load = '{0}%'.format(round(load, 2)) output.append( ' + {0} - port {1} - {2} master(s), {3} slave(s) - capacity: {4}, load: {5}' .format(mds_service.number, mds_service.service.ports[0], masters, slaves, capacity, load)) if vpools_deployed is False: output.append('No vPools deployed') print '\x1b[2J\x1b[H' + '\n'.join(output) time.sleep(1) except KeyboardInterrupt: pass
def get_scrub_storagerouters(self): """ Loads a list of suitable StorageRouters for scrubbing the given vDisk """ storagerouters = [] for storagerouter in StorageRouterList.get_storagerouters(): scrub_partitions = storagerouter.partition_config.get(DiskPartition.ROLES.SCRUB, []) if len(scrub_partitions) == 0: continue storagerouters.append(storagerouter) return storagerouters
def _remote_stack(self): """ Live list of information about remote linked OSDs of type ALBA BACKEND :return: Information about all linked OSDs :rtype: dict """ # Import here to prevent from circular references from ovs.dal.hybrids.albaosd import AlbaOSD def _load_backend_info(_connection_info, _alba_backend_guid): client = OVSClient(ip=_connection_info['host'], port=_connection_info['port'], credentials=(_connection_info['username'], _connection_info['password']), version=3) try: info = client.get('/alba/backends/{0}/'.format(_alba_backend_guid), params={'contents': 'local_summary'}) with lock: return_value[_alba_backend_guid].update(info['local_summary']) except NotFoundException: return_value[_alba_backend_guid]['error'] = 'backend_deleted' except ForbiddenException: return_value[_alba_backend_guid]['error'] = 'not_allowed' except Exception as ex: return_value[_alba_backend_guid]['error'] = 'unknown' AlbaBackend._logger.exception('Collecting remote ALBA backend information failed with error: {0}'.format(ex)) # Retrieve local summaries of all related OSDs of type ALBA_BACKEND lock = Lock() threads = [] return_value = {} cluster_ips = [sr.ip for sr in StorageRouterList.get_storagerouters()] for osd in self.osds: if osd.osd_type == AlbaOSD.OSD_TYPES.ALBA_BACKEND and osd.metadata is not None: backend_info = osd.metadata['backend_info'] connection_info = osd.metadata['backend_connection_info'] connection_host = connection_info['host'] alba_backend_guid = backend_info['linked_guid'] return_value[alba_backend_guid] = {'name': backend_info['linked_name'], 'error': '', 'domain': None if osd.domain is None else {'guid': osd.domain_guid, 'name': osd.domain.name}, 'preset': backend_info['linked_preset'], 'osd_id': backend_info['linked_alba_id'], 'local_ip': connection_host in cluster_ips, 'remote_host': connection_host} thread = Thread(target=_load_backend_info, args=(connection_info, alba_backend_guid)) thread.start() threads.append(thread) for thread in threads: thread.join() return return_value
def check_scrub_partition_present(): """ Checks whether at least 1 scrub partition is present on any StorageRouter :return: True if at least 1 SCRUB role present in the cluster else False :rtype: bool """ for storage_router in StorageRouterList.get_storagerouters(): for disk in storage_router.disks: for partition in disk.partitions: if DiskPartition.ROLES.SCRUB in partition.roles: return True return False
def list(self, query=None): """ Overview of all Storage Routers """ if query is None: return StorageRouterList.get_storagerouters() else: query = json.loads(query) query_result = DataList({'object': StorageRouter, 'data': DataList.select.GUIDS, 'query': query}).data return DataObjectList(query_result, StorageRouter)
def refresh_package_information(): """ Retrieve and store the package information of all StorageRouters :return: None """ GenericController._logger.info('Updating package information') threads = [] information = {} all_storagerouters = StorageRouterList.get_storagerouters() for storagerouter in all_storagerouters: information[storagerouter.ip] = {} for fct in Toolbox.fetch_hooks('update', 'get_package_info_multi'): try: # We make use of these clients in Threads --> cached = False client = SSHClient(endpoint=storagerouter, username='******', cached=False) except UnableToConnectException: information[storagerouter.ip]['errors'] = [ 'StorageRouter {0} is inaccessible'.format( storagerouter.name) ] break thread = Thread(target=fct, args=(client, information)) thread.start() threads.append(thread) for fct in Toolbox.fetch_hooks('update', 'get_package_info_single'): thread = Thread(target=fct, args=(information, )) thread.start() threads.append(thread) for thread in threads: thread.join() errors = [] copy_information = copy.deepcopy(information) for ip, info in information.iteritems(): if len(info.get('errors', [])) > 0: errors.extend( ['{0}: {1}'.format(ip, error) for error in info['errors']]) copy_information.pop(ip) for storagerouter in all_storagerouters: info = copy_information.get(storagerouter.ip, {}) if 'errors' in info: info.pop('errors') storagerouter.package_information = info storagerouter.save() if len(errors) > 0: errors = [str(error) for error in set(errors)] raise Exception(' - {0}'.format('\n - '.join(errors)))
def get_scrub_storagerouters(self): """ Loads a list of suitable StorageRouters for scrubbing the given vDisk """ storagerouters = [] for storagerouter in StorageRouterList.get_storagerouters(): scrub_partitions = storagerouter.partition_config.get( DiskPartition.ROLES.SCRUB, []) if len(scrub_partitions) == 0: continue storagerouters.append(storagerouter) return storagerouters
def install_plugins(): """ (Re)load plugins """ if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')): # If the watcher is running, 'ovs setup' was executed and we need to restart everything to load # the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed from ovs.dal.lists.storagerouterlist import StorageRouterList clients = [] try: for storagerouter in StorageRouterList.get_storagerouters(): clients.append(SSHClient(storagerouter, username='******')) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') for client in clients: for service_name in ['watcher-framework', 'memcached']: ServiceManager.stop_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status( service_name, client=client) is False: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError( 'Could not stop service: {0}'.format(service_name)) for client in clients: for service_name in ['memcached', 'watcher-framework']: ServiceManager.start_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status( service_name, client=client) is True: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError( 'Could not start service: {0}'.format( service_name)) from ovs.dal.helpers import Migration Migration.migrate() from ovs.lib.helpers.toolbox import Toolbox ip = System.get_my_storagerouter().ip functions = Toolbox.fetch_hooks('plugin', 'postinstall') for function in functions: function(ip=ip)
def build_clients(self): # type: () -> Dict[StorageRouter, SSHClient] """ Builds SSHClients towards all StorageRouters :return: SSHClient mapped by storagerouter :rtype: dict((storagerouter, sshclient)) """ clients = {} for storagerouter in StorageRouterList.get_storagerouters(): client = self.build_ssh_client(storagerouter) if client is not None: clients[storagerouter] = client return clients
def get_scrub_storagerouters(self): """ Loads a list of suitable StorageRouters for scrubbing the given vDisk :return: A list of StorageRouters which have the SCRUB role :rtype: list[ovs.dal.hybrids.storagerouter.StorageRouter] """ storagerouters = [] for storagerouter in StorageRouterList.get_storagerouters(): scrub_partitions = storagerouter.partition_config.get(DiskPartition.ROLES.SCRUB, []) if len(scrub_partitions) == 0: continue storagerouters.append(storagerouter) return storagerouters
def get_my_storagerouter(): """ Returns unique machine storagerouter id """ from ovs.dal.hybrids.storagerouter import StorageRouter from ovs.dal.lists.storagerouterlist import StorageRouterList if not System.my_storagerouter_guid: for storagerouter in StorageRouterList.get_storagerouters(): if storagerouter.machine_id == System.get_my_machine_id(): System.my_storagerouter_guid = storagerouter.guid return StorageRouter(System.my_storagerouter_guid)
def get_my_storagerouter(): """ Returns unique machine storagerouter id """ from ovs.dal.hybrids.storagerouter import StorageRouter from ovs.dal.lists.storagerouterlist import StorageRouterList if not System.my_storagerouter_guid: for storagerouter in StorageRouterList.get_storagerouters(): if storagerouter.machine_id == System.get_my_machine_id(): System.my_storagerouter_guid = storagerouter.guid return StorageRouter(System.my_storagerouter_guid)
def listEdgeclients(): edgeclients = [] protocol = getEdgeProtocol() for storagerouter in StorageRouterList.get_storagerouters(): if storagerouter.status == 'FAILURE': continue for storagedriver in storagerouter.storagedrivers: edgeclient = {'vpool': storagedriver.vpool.name, 'protocol': protocol, 'ip': storagerouter.ip, 'port': storagedriver.ports['edge']} edgeclients.append(edgeclient) return edgeclients
def process_response(self, request, response): """ Processes responses """ _ = self # Process CORS responses if 'HTTP_ORIGIN' in request.META: storagerouters = StorageRouterList.get_storagerouters() allowed_origins = ['https://{0}'.format(storagerouter.ip) for storagerouter in storagerouters] if request.META['HTTP_ORIGIN'] in allowed_origins: response['Access-Control-Allow-Origin'] = request.META['HTTP_ORIGIN'] response['Access-Control-Allow-Headers'] = 'x-requested-with, content-type, accept, origin, authorization' response['Access-Control-Allow-Methods'] = 'GET, POST, PUT, PATCH, DELETE, OPTIONS' return response
def collapse_arakoon(): """ Collapse Arakoon's Tlogs :return: None """ ScheduledTaskController._logger.info('Starting arakoon collapse') storagerouters = StorageRouterList.get_storagerouters() cluster_info = [('cacc', storagerouters[0], True)] cluster_names = [] for service in ServiceList.get_services(): if service.is_internal is True and service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON, ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR): cluster = service.name.replace('arakoon-', '') if cluster in cluster_names: continue cluster_names.append(cluster) cluster_info.append((cluster, service.storagerouter, False)) workload = {} for cluster, storagerouter, filesystem in cluster_info: ScheduledTaskController._logger.debug(' Collecting info for cluster {0}'.format(cluster)) config = ArakoonClusterConfig(cluster, filesystem=filesystem) config.load_config(storagerouter.ip) for node in config.nodes: if node.ip not in workload: workload[node.ip] = {'node_id': node.name, 'clusters': []} workload[node.ip]['clusters'].append((cluster, filesystem)) for storagerouter in storagerouters: try: if storagerouter.ip not in workload: continue node_workload = workload[storagerouter.ip] client = SSHClient(storagerouter) for cluster, filesystem in node_workload['clusters']: try: ScheduledTaskController._logger.debug(' Collapsing cluster {0} on {1}'.format(cluster, storagerouter.ip)) if filesystem is True: config_path = ArakoonClusterConfig.CONFIG_FILE.format(cluster) else: config_path = Configuration.get_configuration_path(ArakoonClusterConfig.CONFIG_KEY.format(cluster)) client.run(['arakoon', '--collapse-local', node_workload['node_id'], '2', '-config', config_path]) ScheduledTaskController._logger.info(' Collapsing cluster {0} on {1} completed'.format(cluster, storagerouter.ip)) except: ScheduledTaskController._logger.exception(' Collapsing cluster {0} on {1} failed'.format(cluster, storagerouter.ip)) except UnableToConnectException: ScheduledTaskController._logger.error(' Could not collapse any cluster on {0} (not reachable)'.format(storagerouter.name)) ScheduledTaskController._logger.info('Arakoon collapse finished')
def apply(license_string): """ Applies a license. It will apply as much licenses as possible, however, it won't fail on invalid licenses as it will simply skip them. """ try: clients = {} storagerouters = StorageRouterList.get_storagerouters() try: for storagerouter in storagerouters: clients[storagerouter] = SSHClient(storagerouter.ip) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') data = LicenseController._decode(license_string) for component in data: cdata = data[component] name = cdata['name'] data = cdata['data'] token = cdata['token'] valid_until = float(cdata['valid_until']) if 'valid_until' in cdata else None if valid_until is not None and valid_until <= time.time(): continue signature = cdata['signature'] if 'signature' in cdata else None validate_functions = Toolbox.fetch_hooks('license', '{0}.validate'.format(component)) apply_functions = Toolbox.fetch_hooks('license', '{0}.apply'.format(component)) if len(validate_functions) == 1 and len(apply_functions) == 1: valid, metadata = validate_functions[0](component=component, data=data, signature=signature) if valid is True: success = apply_functions[0](component=component, data=data, signature=signature) if success is True: license_object = LicenseList.get_by_component(component) if license_object is None: license_object = License() license_object.component = component license_object.name = name license_object.token = token license_object.data = data license_object.valid_until = valid_until license_object.signature = signature license_object.save() license_contents = [] for lic in LicenseList.get_licenses(): license_contents.append(lic.hash) for storagerouter in storagerouters: client = clients[storagerouter] client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents))) except Exception, ex: LicenseController._logger.exception('Error applying license: {0}'.format(ex)) return None
def refresh_package_information(): """ Retrieve and store the package information of all StorageRouters :return: None """ GenericController._logger.info('Updating package information') threads = [] information = {} all_storagerouters = StorageRouterList.get_storagerouters() for storagerouter in all_storagerouters: information[storagerouter.ip] = {} for function in Toolbox.fetch_hooks('update', 'get_package_info_multi'): try: # We make use of these clients in Threads --> cached = False client = SSHClient(endpoint=storagerouter, username='******', cached=False) except UnableToConnectException: information[storagerouter.ip]['errors'] = ['StorageRouter {0} is inaccessible'.format(storagerouter.name)] break thread = Thread(target=function, args=(client, information)) thread.start() threads.append(thread) for function in Toolbox.fetch_hooks('update', 'get_package_info_single'): thread = Thread(target=function, args=(information,)) thread.start() threads.append(thread) for thread in threads: thread.join() errors = [] copy_information = copy.deepcopy(information) for ip, info in information.iteritems(): if len(info.get('errors', [])) > 0: errors.extend(['{0}: {1}'.format(ip, error) for error in info['errors']]) copy_information.pop(ip) for storagerouter in all_storagerouters: info = copy_information.get(storagerouter.ip, {}) if 'errors' in info: info.pop('errors') storagerouter.package_information = info storagerouter.save() if len(errors) > 0: errors = [str(error) for error in set(errors)] raise Exception(' - {0}'.format('\n - '.join(errors)))
def _default_starting_values(since=None, until=None, search_locations=None, hosts=None): if until and since is not None and isinstance( since, str) and isinstance(until, str): # Test for times to be properly formatted, allow hh:mm or hh:mm:ss pattern = re.compile( r'(^[2][0-3]|[0-1][0-9]):[0-5][0-9](:[0-5][0-9])?$') # If only hours are supplied, match them to a day if pattern.match(since) or pattern.match(until): # Determine Time Range yesterday = date.fromordinal(date.today().toordinal() - 1).strftime('%Y-%m-%d') today = datetime.now().strftime('%Y-%m-%d') now = datetime.now().strftime('%R') if since > now or since > until: search_start = yesterday else: search_start = today if until > since > now: search_end = yesterday else: search_end = today since = LogFileTimeParser._parse_date(search_start + ' ' + since) until = LogFileTimeParser._parse_date(search_end + ' ' + until) else: # Set dates since = LogFileTimeParser._parse_date(since) until = LogFileTimeParser._parse_date(until) # Setup default times if since is None: since = (datetime.today() - timedelta(hours=1)) if until is None: until = datetime.today() if search_locations is None: execution_mode = LogFileTimeParser._get_execution_mode() search_locations = LogFileTimeParser.STANDARD_SEARCH_LOCATIONS for service, info in LogFileTimeParser.INTERNAL_MAPPING.iteritems( ): search_locations.append(info.get(execution_mode)) if not hosts: hosts = [sr.ip for sr in StorageRouterList.get_storagerouters()] return since, until, search_locations, hosts
def listEdgeclients(): edgeclients = [] protocol = getEdgeProtocol() for storagerouter in StorageRouterList.get_storagerouters(): if storagerouter.status == "FAILURE": continue for storagedriver in storagerouter.storagedrivers: edgeclient = { "vpool": storagedriver.vpool.name, "protocol": protocol, "ip": storagerouter.ip, "port": storagedriver.ports["edge"], } edgeclients.append(edgeclient) return edgeclients
def _storage_router_layout(self): """ Creates a dictionary with information about which Storage Routers use this domain as its normal and recovery domain :return: Information about Storage Routers using this domain :rtype: dict """ layout = {'regular': [], 'recovery': []} for sr in StorageRouterList.get_storagerouters(): for junction in sr.domains: if junction.domain_guid == self.guid: if junction.backup is True: layout['recovery'].append(sr.guid) else: layout['regular'].append(sr.guid) return layout
def install_plugins(): """ (Re)load plugins """ if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')): # If the watcher is running, 'ovs setup' was executed and we need to restart everything to load # the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed from ovs.dal.lists.storagerouterlist import StorageRouterList clients = [] try: for storagerouter in StorageRouterList.get_storagerouters(): clients.append(SSHClient(storagerouter, username='******')) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') for client in clients: for service_name in ['watcher-framework', 'memcached']: ServiceManager.stop_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status(service_name, client=client) is False: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError('Could not stop service: {0}'.format(service_name)) for client in clients: for service_name in ['memcached', 'watcher-framework']: ServiceManager.start_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status(service_name, client=client) is True: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError('Could not start service: {0}'.format(service_name)) from ovs.dal.helpers import Migration Migration.migrate() from ovs.lib.helpers.toolbox import Toolbox ip = System.get_my_storagerouter().ip functions = Toolbox.fetch_hooks('plugin', 'postinstall') for function in functions: function(ip=ip)
def register(name, email, company, phone, newsletter): """ Registers the environment """ SupportAgent().run() # Execute a single heartbeat run client = OVSClient('monitoring.openvstorage.com', 443, credentials=None, verify=True, version=1) task_id = client.post('/support/register/', data={'cluster_id': Configuration.get('ovs.support.cid'), 'name': name, 'email': email, 'company': company, 'phone': phone, 'newsletter': newsletter, 'register_only': True}) if task_id: client.wait_for_task(task_id, timeout=120) for storagerouter in StorageRouterList.get_storagerouters(): client = SSHClient(storagerouter) client.config_set('ovs.core.registered', True)
def _default_starting_values(since=None, until=None, search_locations=None, hosts=None): if until and since is not None and isinstance(since, str) and isinstance(until, str): # Test for times to be properly formatted, allow hh:mm or hh:mm:ss pattern = re.compile(r'(^[2][0-3]|[0-1][0-9]):[0-5][0-9](:[0-5][0-9])?$') # If only hours are supplied, match them to a day if pattern.match(since) or pattern.match(until): # Determine Time Range yesterday = date.fromordinal(date.today().toordinal() - 1).strftime('%Y-%m-%d') today = datetime.now().strftime('%Y-%m-%d') now = datetime.now().strftime('%R') if since > now or since > until: search_start = yesterday else: search_start = today if until > since > now: search_end = yesterday else: search_end = today since = LogFileTimeParser._parse_date(search_start + ' ' + since) until = LogFileTimeParser._parse_date(search_end + ' ' + until) else: # Set dates since = LogFileTimeParser._parse_date(since) until = LogFileTimeParser._parse_date(until) # Setup default times if since is None: since = (datetime.today() - timedelta(hours=1)) if until is None: until = datetime.today() if search_locations is None: execution_mode = LogFileTimeParser._get_execution_mode() search_locations = LogFileTimeParser.STANDARD_SEARCH_LOCATIONS for service, info in LogFileTimeParser.INTERNAL_MAPPING.iteritems(): search_locations.append(info.get(execution_mode)) if not hosts: hosts = [sr.ip for sr in StorageRouterList.get_storagerouters()] return since, until, search_locations, hosts
def sync_with_reality(storagerouter_guid=None): """ Syncs the Disks from all StorageRouters with the reality. :param storagerouter_guid: Guid of the Storage Router to synchronize """ storagerouters = [] if storagerouter_guid is not None: storagerouters.append(StorageRouter(storagerouter_guid)) else: storagerouters = StorageRouterList.get_storagerouters() for storagerouter in storagerouters: try: client = SSHClient(storagerouter, username='******') except UnableToConnectException: DiskController._logger.info('Could not connect to StorageRouter {0}, skipping'.format(storagerouter.ip)) continue configuration = {} # Gather mount data mount_mapping = {} mount_data = client.run('mount') for mount in mount_data.splitlines(): mount = mount.strip() match = re.search('(/dev/(.+?)) on (/.*?) type.*', mount) if match is not None: dev_name = match.groups()[0] uuid = client.run('blkid -o value -s UUID {0}'.format(dev_name)) if uuid: mount_mapping[uuid] = match.groups()[2] else: mount_mapping[match.groups()[1]] = match.groups()[2] # Gather raid information try: md_information = client.run('mdadm --detail /dev/md*', suppress_logging=True) except CalledProcessError: md_information = '' raid_members = [] for member in re.findall('(?: +[0-9]+){4} +[^/]+/dev/([a-z0-9]+)', md_information): raid_members.append(member) # Gather disk information with remote(storagerouter.ip, [Context, os]) as rem: context = rem.Context() devices = [device for device in context.list_devices(subsystem='block') if ('ID_TYPE' in device and device['ID_TYPE'] == 'disk') or ('DEVNAME' in device and ('loop' in device['DEVNAME'] or 'nvme' in device['DEVNAME'] or 'md' in device['DEVNAME']))] for device in devices: is_partition = device['DEVTYPE'] == 'partition' device_path = device['DEVNAME'] device_name = device_path.split('/')[-1] partition_id = None partition_name = None extended_partition_info = None if is_partition is True: partition_name = device['ID_FS_UUID'] if 'ID_FS_UUID' in device else device_name if 'ID_PART_ENTRY_NUMBER' in device: extended_partition_info = True partition_id = device['ID_PART_ENTRY_NUMBER'] if device_name.startswith('nvme') or device_name.startswith('loop'): device_name = device_name[:0 - int(len(partition_id)) - 1] elif device_name.startswith('md'): device_name = device_name[:device_name.index('p')] else: device_name = device_name[:0 - int(len(partition_id))] else: DiskController._logger.debug('Partition {0} has no partition metadata'.format(device_path)) extended_partition_info = False match = re.match('^(\D+?)(\d+)$', device_name) if match is None: DiskController._logger.debug('Could not handle disk/partition {0}'.format(device_path)) continue # Unable to handle this disk/partition partition_id = match.groups()[1] device_name = match.groups()[0] sectors = int(client.run('cat /sys/block/{0}/size'.format(device_name))) sector_size = int(client.run('cat /sys/block/{0}/queue/hw_sector_size'.format(device_name))) rotational = int(client.run('cat /sys/block/{0}/queue/rotational'.format(device_name))) if sectors == 0: continue if device_name in raid_members: continue if device_name not in configuration: configuration[device_name] = {'partitions': {}} path = None for path_type in ['by-id', 'by-uuid']: if path is not None: break if 'DEVLINKS' in device: for item in device['DEVLINKS'].split(' '): if path_type in item: path = item if path is None: path = device_path if is_partition is True: if 'ID_PART_ENTRY_TYPE' in device and device['ID_PART_ENTRY_TYPE'] == '0x5': continue # This is an extended partition, let's skip that one if extended_partition_info is True: offset = int(device['ID_PART_ENTRY_OFFSET']) * sector_size size = int(device['ID_PART_ENTRY_SIZE']) * sector_size else: match = re.match('^(\D+?)(\d+)$', device_path) if match is None: DiskController._logger.debug('Could not handle disk/partition {0}'.format(device_path)) continue # Unable to handle this disk/partition partitions_info = DiskTools.get_partitions_info(match.groups()[0]) if device_path in partitions_info: partition_info = partitions_info[device_path] offset = int(partition_info['start']) size = int(partition_info['size']) else: DiskController._logger.warning('Could not retrieve partition info for disk/partition {0}'.format(device_path)) continue configuration[device_name]['partitions'][partition_id] = {'offset': offset, 'size': size, 'path': path, 'state': 'OK'} partition_data = configuration[device_name]['partitions'][partition_id] if partition_name in mount_mapping: mountpoint = mount_mapping[partition_name] partition_data['mountpoint'] = mountpoint partition_data['inode'] = rem.os.stat(mountpoint).st_dev del mount_mapping[partition_name] try: client.run('touch {0}/{1}; rm {0}/{1}'.format(mountpoint, str(time.time()))) except CalledProcessError: partition_data['state'] = 'FAILURE' pass if 'ID_FS_TYPE' in device: partition_data['filesystem'] = device['ID_FS_TYPE'] else: configuration[device_name].update({'name': device_name, 'path': path, 'vendor': device['ID_VENDOR'] if 'ID_VENDOR' in device else None, 'model': device['ID_MODEL'] if 'ID_MODEL' in device else None, 'size': sector_size * sectors, 'is_ssd': rotational == 0, 'state': 'OK'}) for partition_name in mount_mapping: device_name = partition_name.split('/')[-1] match = re.search('^(\D+?)(\d+)$', device_name) if match is not None: device_name = match.groups()[0] partition_id = match.groups()[1] if device_name not in configuration: configuration[device_name] = {'partitions': {}, 'state': 'MISSING'} configuration[device_name]['partitions'][partition_id] = {'mountpoint': mount_mapping[partition_name], 'state': 'MISSING'} # Sync the model disk_names = [] for disk in storagerouter.disks: if disk.name not in configuration: for partition in disk.partitions: partition.delete() disk.delete() else: disk_names.append(disk.name) DiskController._update_disk(disk, configuration[disk.name]) partitions = [] partition_info = configuration[disk.name]['partitions'] for partition in disk.partitions: if partition.id not in partition_info: partition.delete() else: partitions.append(partition.id) DiskController._update_partition(partition, partition_info[partition.id]) for partition_id in partition_info: if partition_id not in partitions: DiskController._create_partition(partition_id, partition_info[partition_id], disk) for disk_name in configuration: if disk_name not in disk_names and configuration[disk_name]['state'] not in ['MISSING']: disk = Disk() disk.storagerouter = storagerouter disk.name = disk_name DiskController._update_disk(disk, configuration[disk_name]) partition_info = configuration[disk_name]['partitions'] for partition_id in partition_info: if partition_info[partition_id]['state'] not in ['MISSING']: DiskController._create_partition(partition_id, partition_info[partition_id], disk)
def _bootstrap_dal_models(self): """ Load/hook dal models as snmp oids """ _guids = set() enabled_key = "{0}_config_dal_enabled".format(STORAGE_PREFIX) self.instance_oid = 0 try: enabled = self.persistent.get(enabled_key) except KeyNotFoundException: enabled = True # Enabled by default, can be disabled by setting the key if enabled: from ovs.dal.lists.vdisklist import VDiskList from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.dal.lists.pmachinelist import PMachineList from ovs.dal.lists.vmachinelist import VMachineList from ovs.dal.lists.vpoollist import VPoolList from ovs.dal.lists.storagedriverlist import StorageDriverList for storagerouter in StorageRouterList.get_storagerouters(): _guids.add(storagerouter.guid) if not self._check_added(storagerouter): self._register_dal_model(10, storagerouter, 'guid', "0") self._register_dal_model(10, storagerouter, 'name', "1") self._register_dal_model(10, storagerouter, 'pmachine', "3", key = 'host_status') self._register_dal_model(10, storagerouter, 'description', "4") self._register_dal_model(10, storagerouter, 'devicename', "5") self._register_dal_model(10, storagerouter, 'dtl_mode', "6") self._register_dal_model(10, storagerouter, 'ip', "8") self._register_dal_model(10, storagerouter, 'machineid', "9") self._register_dal_model(10, storagerouter, 'status', "10") self._register_dal_model(10, storagerouter, '#vdisks', "11", func = lambda storagerouter: len([vdisk for vpool_vdisks in [storagedriver.vpool.vdisks for storagedriver in storagerouter.storagedrivers] for vdisk in vpool_vdisks if vdisk.storagedriver_id == storagedriver.storagedriver_id]), atype = int) self._register_dal_model(10, storagerouter, '#vmachines', "12", func = lambda storagerouter: len(set([vdisk.vmachine.guid for vpool_vdisks in [storagedriver.vpool.vdisks for storagedriver in storagerouter.storagedrivers] for vdisk in vpool_vdisks if vdisk.storagedriver_id == storagedriver.storagedriver_id])), atype = int) self._register_dal_model(10, storagerouter, '#stored_data', "13", func = lambda storagerouter: sum([vdisk.vmachine.stored_data for vpool_vdisks in [storagedriver.vpool.vdisks for storagedriver in storagerouter.storagedrivers] for vdisk in vpool_vdisks if vdisk.storagedriver_id == storagedriver.storagedriver_id]), atype = int) self.instance_oid += 1 for vm in VMachineList.get_vmachines(): _guids.add(vm.guid) if not self._check_added(vm): if vm.is_vtemplate: self._register_dal_model(11, vm, 'guid', "0") self._register_dal_model(11, vm, 'name', "1") def _children(vmt): children = 0 disks = [vd.guid for vd in vmt.vdisks] for vdisk in [vdisk.parent_vdisk_guid for item in [vm.vdisks for vm in VMachineList.get_vmachines() if not vm.is_vtemplate] for vdisk in item]: for disk in disks: if vdisk == disk: children += 1 return children self._register_dal_model(11, vm, '#children', 2, func = _children, atype = int) self.instance_oid += 1 for vm in VMachineList.get_vmachines(): _guids.add(vm.guid) if not self._check_added(vm): if not vm.is_vtemplate: self._register_dal_model(0, vm, 'guid', "0") self._register_dal_model(0, vm, 'name', "1") self._register_dal_model(0, vm, 'statistics', "2.0", key = "operations", atype = int) self._register_dal_model(0, vm, 'statistics', "2.1", key = "cluster_cache_misses_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.2", key = "data_read", atype = int) self._register_dal_model(0, vm, 'statistics', "2.3", key = "sco_cache_misses", atype = int) self._register_dal_model(0, vm, 'statistics', "2.4", key = "sco_cache_hits_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.5", key = "sco_cache_hits", atype = int) self._register_dal_model(0, vm, 'statistics', "2.6", key = "write_operations", atype = int) self._register_dal_model(0, vm, 'statistics', "2.7", key = "cluster_cache_misses", atype = int) self._register_dal_model(0, vm, 'statistics', "2.8", key = "read_operations_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.9", key = "sco_cache_misses_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.10", key = "backend_write_operations", atype = int) self._register_dal_model(0, vm, 'statistics', "2.11", key = "backend_data_read", atype = int) self._register_dal_model(0, vm, 'statistics', "2.12", key = "cache_hits", atype = int) self._register_dal_model(0, vm, 'statistics', "2.13", key = "backend_write_operations_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.14", key = "metadata_store_hits_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.15", key = "metadata_store_misses", atype = int) self._register_dal_model(0, vm, 'statistics', "2.16", key = "backend_data_written", atype = int) self._register_dal_model(0, vm, 'statistics', "2.17", key = "data_read_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.18", key = "read_operations", atype = int) self._register_dal_model(0, vm, 'statistics', "2.19", key = "cluster_cache_hits", atype = int) self._register_dal_model(0, vm, 'statistics', "2.20", key = "data_written_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.21", key = "cluster_cache_hits_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.22", key = "cache_hits_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.23", key = "timestamp", atype = int) self._register_dal_model(0, vm, 'statistics', "2.24", key = "metadata_store_misses_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.25", key = "backend_data_written_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.26", key = "backend_read_operations", atype = int) self._register_dal_model(0, vm, 'statistics', "2.27", key = "data_written", atype = int) self._register_dal_model(0, vm, 'statistics', "2.28", key = "metadata_store_hits", atype = int) self._register_dal_model(0, vm, 'statistics', "2.29", key = "backend_data_read_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.30", key = "operations_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.31", key = "backend_read_operations_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.32", key = "data_transferred_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.33", key = "write_operations_ps", atype = int) self._register_dal_model(0, vm, 'statistics', "2.34", key = "data_transferred", atype = int) self._register_dal_model(0, vm, 'stored_data', "3", atype = int) self._register_dal_model(0, vm, 'description', "4") self._register_dal_model(0, vm, 'devicename', "5") self._register_dal_model(0, vm, 'dtl_mode', "6") self._register_dal_model(0, vm, 'hypervisorid', "7") self._register_dal_model(0, vm, 'ip', "8") self._register_dal_model(0, vm, 'status', "10") self._register_dal_model(0, vm, 'stored_data', "10", atype = int) self._register_dal_model(0, vm, 'snapshots', "11", atype = int) self._register_dal_model(0, vm, 'vdisks', "12", atype = int) self._register_dal_model(0, vm, 'DTL', '13', func = lambda vm: 'DEGRADED' if all(item == 'DEGRADED' for item in [vd.info['failover_mode'] for vd in vm.vdisks]) else 'OK') self.instance_oid += 1 for vd in VDiskList.get_vdisks(): _guids.add(vd.guid) if not self._check_added(vd): self._register_dal_model(1, vd, 'guid', "0") self._register_dal_model(1, vd, 'name', "1") self._register_dal_model(1, vd, 'statistics', "2.0", key = "operations", atype = int) self._register_dal_model(1, vd, 'statistics', "2.1", key = "data_written_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.2", key = "data_read", atype = int) self._register_dal_model(1, vd, 'statistics', "2.3", key = "sco_cache_misses", atype = int) self._register_dal_model(1, vd, 'statistics', "2.4", key = "sco_cache_hits_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.5", key = "sco_cache_hits", atype = int) self._register_dal_model(1, vd, 'statistics', "2.6", key = "write_operations", atype = int) self._register_dal_model(1, vd, 'statistics', "2.7", key = "cluster_cache_misses", atype = int) self._register_dal_model(1, vd, 'statistics', "2.8", key = "read_operations_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.9", key = "sco_cache_misses_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.10", key = "backend_write_operations", atype = int) self._register_dal_model(1, vd, 'statistics', "2.11", key = "backend_data_read", atype = int) self._register_dal_model(1, vd, 'statistics', "2.12", key = "cache_hits", atype = int) self._register_dal_model(1, vd, 'statistics', "2.13", key = "backend_write_operations_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.14", key = "metadata_store_hits_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.15", key = "metadata_store_misses", atype = int) self._register_dal_model(1, vd, 'statistics', "2.16", key = "backend_data_written", atype = int) self._register_dal_model(1, vd, 'statistics', "2.17", key = "data_read_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.18", key = "read_operations", atype = int) self._register_dal_model(1, vd, 'statistics', "2.19", key = "cluster_cache_hits", atype = int) self._register_dal_model(1, vd, 'statistics', "2.20", key = "cluster_cache_misses_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.21", key = "cluster_cache_hits_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.22", key = "cache_hits_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.23", key = "timestamp", atype = int) self._register_dal_model(1, vd, 'statistics', "2.24", key = "metadata_store_misses_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.25", key = "backend_data_written_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.26", key = "backend_read_operations", atype = int) self._register_dal_model(1, vd, 'statistics', "2.27", key = "data_written", atype = int) self._register_dal_model(1, vd, 'statistics', "2.28", key = "metadata_store_hits", atype = int) self._register_dal_model(1, vd, 'statistics', "2.29", key = "backend_data_read_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.30", key = "operations_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.31", key = "backend_read_operations_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.32", key = "data_transferred_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.33", key = "write_operations_ps", atype = int) self._register_dal_model(1, vd, 'statistics', "2.34", key = "data_transferred", atype = int) self._register_dal_model(1, vd, 'info', "3", key = 'stored', atype = int) self._register_dal_model(1, vd, 'info', "4", key = 'failover_mode', atype = int) self._register_dal_model(1, vd, 'snapshots', "5", atype = int) self.instance_oid += 1 for pm in PMachineList.get_pmachines(): _guids.add(pm.guid) if not self._check_added(pm): self._register_dal_model(2, pm, 'guid', "0") self._register_dal_model(2, pm, 'name', "1") self._register_dal_model(2, pm, 'host_status', "2") self.instance_oid += 1 for vp in VPoolList.get_vpools(): _guids.add(vp.guid) if not self._check_added(vp): self._register_dal_model(3, vp, 'guid', "0") self._register_dal_model(3, vp, 'name', "1") self._register_dal_model(3, vp, 'statistics', "2.0", key = "operations", atype = int) self._register_dal_model(3, vp, 'statistics', "2.1", key = "cluster_cache_misses_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.2", key = "data_read", atype = int) self._register_dal_model(3, vp, 'statistics', "2.3", key = "sco_cache_misses", atype = int) self._register_dal_model(3, vp, 'statistics', "2.4", key = "sco_cache_hits_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.5", key = "sco_cache_hits", atype = int) self._register_dal_model(3, vp, 'statistics', "2.6", key = "write_operations", atype = int) self._register_dal_model(3, vp, 'statistics', "2.7", key = "cluster_cache_misses", atype = int) self._register_dal_model(3, vp, 'statistics', "2.8", key = "read_operations_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.9", key = "sco_cache_misses_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.10", key = "backend_write_operations", atype = int) self._register_dal_model(3, vp, 'statistics', "2.11", key = "backend_data_read", atype = int) self._register_dal_model(3, vp, 'statistics', "2.12", key = "cache_hits", atype = int) self._register_dal_model(3, vp, 'statistics', "2.13", key = "backend_write_operations_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.14", key = "metadata_store_hits_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.15", key = "metadata_store_misses", atype = int) self._register_dal_model(3, vp, 'statistics', "2.16", key = "backend_data_written", atype = int) self._register_dal_model(3, vp, 'statistics', "2.17", key = "data_read_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.18", key = "read_operations", atype = int) self._register_dal_model(3, vp, 'statistics', "2.19", key = "cluster_cache_hits", atype = int) self._register_dal_model(3, vp, 'statistics', "2.20", key = "data_written_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.21", key = "cluster_cache_hits_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.22", key = "cache_hits_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.23", key = "timestamp", atype = int) self._register_dal_model(3, vp, 'statistics', "2.24", key = "metadata_store_misses_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.25", key = "backend_data_written_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.26", key = "backend_read_operations", atype = int) self._register_dal_model(3, vp, 'statistics', "2.27", key = "data_written", atype = int) self._register_dal_model(3, vp, 'statistics', "2.28", key = "metadata_store_hits", atype = int) self._register_dal_model(3, vp, 'statistics', "2.29", key = "backend_data_read_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.30", key = "operations_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.31", key = "backend_read_operations_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.32", key = "data_transferred_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.33", key = "write_operations_ps", atype = int) self._register_dal_model(3, vp, 'statistics', "2.34", key = "data_transferred", atype = int) self._register_dal_model(3, vp, 'status', "3") self._register_dal_model(3, vp, 'description', "4") self._register_dal_model(3, vp, 'vdisks', "5", atype = int) self._register_dal_model(3, vp, '#vmachines', "6", func = lambda vp: len(set([vd.vmachine.guid for vd in vp.vdisks])), atype = int) self.instance_oid += 1 for storagedriver in StorageDriverList.get_storagedrivers(): _guids.add(storagedriver.guid) if not self._check_added(storagedriver): self._register_dal_model(4, storagedriver, 'guid', "0") self._register_dal_model(4, storagedriver, 'name', "1") self._register_dal_model(4, storagedriver, 'stored_data', "2", atype = int) self.instance_oid += 1 try: # try to load OVS Backends from ovs.dal.lists.albabackendlist import AlbaBackendList for backend in AlbaBackendList.get_albabackends(): _guids.add(backend.guid) if not self._check_added(backend): self._register_dal_model(5, backend, 'guid', 0) self._register_dal_model(5, backend, 'name', 1) for disk_id in range(len((backend.all_disks))): self._register_dal_model(5, backend, 'all_disks', '2.{0}.0'.format(disk_id), key = "name", index=disk_id) self._register_dal_model(5, backend, 'all_disks', '2.{0}.1'.format(disk_id), key = "usage.size", atype = long, index=disk_id) self._register_dal_model(5, backend, 'all_disks', '2.{0}.2'.format(disk_id), key = "usage.used", atype = long, index=disk_id) self._register_dal_model(5, backend, 'all_disks', '2.{0}.3'.format(disk_id), key = "usage.available", atype = long, index=disk_id) self._register_dal_model(5, backend, 'all_disks', '2.{0}.4'.format(disk_id), key = "state.state", index=disk_id) self._register_dal_model(5, backend, 'all_disks', '2.{0}.5'.format(disk_id), key = "node_id", index=disk_id) self.instance_oid += 1 except ImportError: print('OVS Backend not present') pass reload = False for object_guid in list(self.model_oids): if object_guid not in _guids: self.model_oids.remove(object_guid) reload = True if reload: self._reload_snmp()
def sync_with_reality(storagerouter_guid=None): """ Syncs the Disks from all StorageRouters with the reality. """ storagerouters = [] if storagerouter_guid is not None: storagerouters.append(StorageRouter(storagerouter_guid)) else: storagerouters = StorageRouterList.get_storagerouters() for storagerouter in storagerouters: try: client = SSHClient(storagerouter, username='******') except UnableToConnectException: logger.info('Could not connect to StorageRouter {0}, skipping'.format(storagerouter.ip)) continue configuration = {} # Gather mount data mount_mapping = {} mount_data = client.run('mount') for mount in mount_data.splitlines(): mount = mount.strip() match = re.search('/dev/(.+?) on (/.*?) type.*', mount) if match is not None: mount_mapping[match.groups()[0]] = match.groups()[1] # Gather disk information with Remote(storagerouter.ip, [Context, os]) as remote: context = remote.Context() devices = [device for device in context.list_devices(subsystem='block') if 'ID_TYPE' in device and device['ID_TYPE'] == 'disk'] for device in devices: is_partition = device['DEVTYPE'] == 'partition' device_path = device['DEVNAME'] device_name = device_path.split('/')[-1] partition_id = None partition_name = None extended_parition_info = None if is_partition is True: if 'ID_PART_ENTRY_NUMBER' in device: extended_parition_info = True partition_id = device['ID_PART_ENTRY_NUMBER'] partition_name = device_name device_name = device_name[:0 - int(len(partition_id))] else: logger.debug('Partition {0} has no partition metadata'.format(device_path)) extended_parition_info = False match = re.match('^(\D+?)(\d+)$', device_name) if match is None: logger.debug('Could not handle disk/partition {0}'.format(device_path)) continue # Unable to handle this disk/partition partition_name = device_name partition_id = match.groups()[1] device_name = match.groups()[0] if device_name not in configuration: configuration[device_name] = {'partitions': {}} path = None for path_type in ['by-id', 'by-uuid']: if path is not None: break for item in device['DEVLINKS'].split(' '): if path_type in item: path = item if path is None: path = device_path sectors = int(client.run('cat /sys/block/{0}/size'.format(device_name))) sector_size = int(client.run('cat /sys/block/{0}/queue/hw_sector_size'.format(device_name))) rotational = int(client.run('cat /sys/block/{0}/queue/rotational'.format(device_name))) if is_partition is True: if 'ID_PART_ENTRY_TYPE' in device and device['ID_PART_ENTRY_TYPE'] == '0x5': continue # This is an extended partition, let's skip that one if extended_parition_info is True: offset = int(device['ID_PART_ENTRY_OFFSET']) * sector_size size = int(device['ID_PART_ENTRY_SIZE']) * sector_size else: match = re.match('^(\D+?)(\d+)$', device_path) if match is None: logger.debug('Could not handle disk/partition {0}'.format(device_path)) continue # Unable to handle this disk/partition fdisk_info = client.run('fdisk -l {0} | grep {1}'.format(match.groups()[0], device_path)).strip() fdisk_data = filter(None, fdisk_info.split(' ')) offset = int(fdisk_data[1]) * sector_size size = (int(fdisk_data[2]) - int(fdisk_data[1])) * sector_size configuration[device_name]['partitions'][partition_id] = {'offset': offset, 'size': size, 'path': path, 'state': 'OK'} partition_data = configuration[device_name]['partitions'][partition_id] if partition_name in mount_mapping: mountpoint = mount_mapping[partition_name] partition_data['mountpoint'] = mountpoint partition_data['inode'] = remote.os.stat(mountpoint).st_dev del mount_mapping[partition_name] try: client.run('touch {0}/{1}; rm {0}/{1}'.format(mountpoint, str(time.time()))) except CalledProcessError: partition_data['state'] = 'FAILURE' pass if 'ID_FS_TYPE' in device: partition_data['filesystem'] = device['ID_FS_TYPE'] else: configuration[device_name].update({'name': device_name, 'path': path, 'vendor': device['ID_VENDOR'] if 'ID_VENDOR' in device else None, 'model': device['ID_MODEL'] if 'ID_MODEL' in device else None, 'size': sector_size * sectors, 'is_ssd': rotational == 0, 'state': 'OK'}) for partition_name in mount_mapping: device_name = partition_name.split('/')[-1] match = re.search('^(\D+?)(\d+)$', device_name) if match is not None: device_name = match.groups()[0] partition_id = match.groups()[1] if device_name not in configuration: configuration[device_name] = {'partitions': {}, 'state': 'MISSING'} configuration[device_name]['partitions'][partition_id] = {'mountpoint': mount_mapping[partition_name], 'state': 'MISSING'} # Sync the model disk_names = [] for disk in storagerouter.disks: if disk.name not in configuration: for partition in disk.partitions: partition.delete() disk.delete() else: disk_names.append(disk.name) DiskController._update_disk(disk, configuration[disk.name]) partitions = [] partition_info = configuration[disk.name]['partitions'] for partition in disk.partitions: if partition.id not in partition_info: partition.delete() else: partitions.append(partition.id) DiskController._update_partition(partition, partition_info[partition.id]) for partition_id in partition_info: if partition_id not in partitions: DiskController._create_partition(partition_id, partition_info[partition_id], disk) for disk_name in configuration: if disk_name not in disk_names and configuration[disk_name]['state'] not in ['MISSING']: disk = Disk() disk.storagerouter = storagerouter disk.name = disk_name DiskController._update_disk(disk, configuration[disk_name]) partition_info = configuration[disk_name]['partitions'] for partition_id in partition_info: if partition_info[partition_id]['state'] not in ['MISSING']: DiskController._create_partition(partition_id, partition_info[partition_id], disk)