def get_client(client_type=None): """ Returns a volatile storage client """ if not hasattr(VolatileFactory, 'store') or VolatileFactory.store is None: if client_type is None: client_type = Configuration.get('ovs.core.storage.volatile') VolatileFactory.store = None if client_type == 'memcache': from ovs.extensions.storage.volatile.memcachestore import MemcacheStore memcache_servers = list() memcache_config = RawConfigParser() memcache_config.read(os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg')) nodes = [node.strip() for node in memcache_config.get('main', 'nodes').split(',')] nodes.sort() for node in nodes: location = memcache_config.get(node, 'location') memcache_servers.append(location) VolatileFactory.store = MemcacheStore(memcache_servers) if client_type == 'default': from ovs.extensions.storage.volatile.dummystore import DummyVolatileStore VolatileFactory.store = DummyVolatileStore() if VolatileFactory.store is None: raise RuntimeError('Invalid client_type specified') return VolatileFactory.store
def get_client(client_type=None): """ Returns a volatile storage client """ if not hasattr(VolatileFactory, 'store') or VolatileFactory.store is None: if client_type is None: client_type = Configuration.get('ovs.core.storage.volatile') VolatileFactory.store = None if client_type == 'memcache': from ovs.extensions.storage.volatile.memcachestore import MemcacheStore memcache_servers = list() memcache_config = ConfigObj( os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg')) nodes = memcache_config.get('main')['nodes'] if type( memcache_config.get('main')['nodes']) == list else [ memcache_config.get('main')['nodes'] ] nodes = [node.strip() for node in nodes] nodes.sort() for node in nodes: location = memcache_config.get(node)['location'] memcache_servers.append(location) VolatileFactory.store = MemcacheStore(memcache_servers) if client_type == 'default': from ovs.extensions.storage.volatile.dummystore import DummyVolatileStore VolatileFactory.store = DummyVolatileStore() if VolatileFactory.store is None: raise RuntimeError('Invalid client_type specified') return VolatileFactory.store
def get_physical_metadata(files, storagerouter_guid): """ Gets physical information about the machine this task is running on """ from ovs.lib.vpool import VPoolController storagerouter = StorageRouter(storagerouter_guid) mountpoints = check_output('mount -v', shell=True).strip().split('\n') mountpoints = [p.split(' ')[2] for p in mountpoints if len(p.split(' ')) > 2 and not p.split(' ')[2].startswith('/dev') and not p.split(' ')[2].startswith('/proc') and not p.split(' ')[2].startswith('/sys') and not p.split(' ')[2].startswith('/run') and p.split(' ')[2] != '/'] arakoon_mountpoint = Configuration.get('ovs.core.db.arakoon.location') if arakoon_mountpoint in mountpoints: mountpoints.remove(arakoon_mountpoint) if storagerouter.pmachine.hvtype == 'KVM': ipaddresses = ['127.0.0.1'] else: ip_path = Configuration.get('ovs.core.ip.path') if ip_path is None: ip_path = "`which ip`" ipaddresses = check_output("{0} a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1".format(ip_path), shell=True).strip().split('\n') ipaddresses = [ip.strip() for ip in ipaddresses] ipaddresses.remove('127.0.0.1') allow_vpool = VPoolController.can_be_served_on(storagerouter_guid) file_existence = {} for check_file in files: file_existence[check_file] = os.path.exists(check_file) and os.path.isfile(check_file) return {'mountpoints': mountpoints, 'ipaddresses': ipaddresses, 'files': file_existence, 'allow_vpool': allow_vpool}
def __init__(self): self._config_corefile = os.path.join( Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-core.conf') self._config_exportfile = os.path.join( Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-export.conf')
def __init__(self): """ Initializes the client """ self._endpoint = Configuration.get('ovs.support.endpoint') self._api = Configuration.get('ovs.support.api') self._enable_support = int(Configuration.get('ovs.support.enablesupport')) > 0 self.interval = int(Configuration.get('ovs.support.interval')) self._url = 'https://{0}/{1}'.format(self._endpoint, self._api)
def __init__(self, vpool_name): self._vpool = vpool_name self._config_specfile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'templates', 'volumedriverfs.json') if not os.path.exists(os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools')): os.makedirs(os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools')) self._config_file = os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools', '{}.json'.format(vpool_name)) self._config_tmpfile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools', '{}.json.tmp'.format(vpool_name)) self._config_readfile_handler = None self._config_file_handler = None self._config_specfile_handler = None self._config_file_content = None
def setUpClass(cls): ArakoonInstaller.ARAKOON_CONFIG_DIR = '/tmp/cfg' ArakoonInstaller.ARAKOON_CONFIG_FILE = '/tmp/cfg/{0}/{0}.cfg' TestArakoonInstaller.expected_global = '[global]\ncluster_id = {0}\ncluster = {1}\nplugins = \n\n' TestArakoonInstaller.expected_base = '[{0}]\nname = {0}\nip = {1}\nclient_port = {2}\nmessaging_port = {3}\ntlog_compression = snappy\nlog_level = info\nlog_dir = /var/log/arakoon/one\nhome = /tmp/db/arakoon/one\ntlog_dir = /tmp/db/tlogs/one\nfsync = true\n\n' # SSH client def _load(ip, password=None): _client = SSHClient._load(ip, password) _client._ip = ip return _client SSHClient._load = staticmethod(SSHClient.load) SSHClient.load = staticmethod(_load) # System def _get_my_machine_id(_client): return TestArakoonInstaller.nodes[_client._ip] def _read_remote_config(_client, _key): return Configuration.get(_key) System.get_my_machine_id = staticmethod(_get_my_machine_id) System.read_remote_config = staticmethod(_read_remote_config) # Configuration def _get(key): if key == 'ovs.core.storage.persistent': return 'arakoon' if key == 'ovs.core.db.arakoon.clusterid': return 'ovsdb' c = PersistentFactory.get_client() if c.exists(key): return c.get(key) return None def _get_int(key): return int(Configuration.get(key)) def _set(key, value): c = PersistentFactory.get_client() c.set(key, value) Configuration.get = staticmethod(_get) Configuration.getInt = staticmethod(_get_int) Configuration.set = staticmethod(_set) Configuration.set('ovs.ports.arakoon', 22000) Configuration.set('ovs.core.db.arakoon.location', '/tmp/db')
def get_heartbeat_data(): """ Returns heartbeat data """ data = {'cid': Configuration.get('ovs.support.cid'), 'nid': Configuration.get('ovs.support.nid'), 'metadata': {}, 'errors': []} try: # Versions data['metadata']['versions'] = Package.get_versions() except Exception, ex: data['errors'].append(str(ex))
def get_mds_storagedriver_config_set(vpool): """ Builds a configuration for all StorageRouters from a given VPool with following goals: * Primary MDS is the local one * All slaves are on different hosts * Maximum `mds.safety` nodes are returned """ mds_per_storagerouter = {} mds_per_load = {} for storagedriver in vpool.storagedrivers: storagerouter = storagedriver.storagerouter mds_service, load = MDSServiceController.get_preferred_mds(storagerouter, vpool, include_load=True) mds_per_storagerouter[storagerouter.guid] = {'host': storagerouter.ip, 'port': mds_service.service.ports[0]} if load not in mds_per_load: mds_per_load[load] = [] mds_per_load[load].append(storagerouter.guid) safety = Configuration.getInt('ovs.storagedriver.mds.safety') config_set = {} for storagerouter_guid in mds_per_storagerouter: config_set[storagerouter_guid] = [mds_per_storagerouter[storagerouter_guid]] for load in sorted(mds_per_load.keys()): if len(config_set[storagerouter_guid]) >= safety: break sr_guids = mds_per_load[load] random.shuffle(sr_guids) for sr_guid in sr_guids: if len(config_set[storagerouter_guid]) >= safety: break if sr_guid != storagerouter_guid: config_set[storagerouter_guid].append(mds_per_storagerouter[sr_guid]) return config_set
def __init__(self, config_type, vpool_name, number=None): """ Initializes the class """ def make_configure(sct): """ section closure """ return lambda **kwargs: self._add(sct, **kwargs) if config_type not in ['storagedriver', 'metadataserver']: raise RuntimeError('Invalid configuration type. Allowed: storagedriver, metadataserver') self.config_type = config_type self.vpool_name = vpool_name self.configuration = {} self.is_new = True self.number = number self.params = copy.deepcopy(StorageDriverConfiguration.parameters) # Never use parameters directly self.base_path = '{0}/storagedriver/{1}'.format(Configuration.get('ovs.core.cfgdir'), self.config_type) if self.number is None: self.path = '{0}/{1}.json'.format(self.base_path, self.vpool_name) else: self.path = '{0}/{1}_{2}.json'.format(self.base_path, self.vpool_name, self.number) # Fix some manual "I know what I'm doing" overrides backend_connection_manager = 'backend_connection_manager' self.params[self.config_type][backend_connection_manager]['optional'].append('s3_connection_strict_consistency') # Generate configure_* methods for section in self.params[self.config_type]: setattr(self, 'configure_{0}'.format(section), make_configure(section))
def run_event_consumer(): """ Check whether to run the event consumer """ rmq_config = RawConfigParser() rmq_config.read(os.path.join(Configuration.get("ovs.core.cfgdir"), "rabbitmqclient.cfg")) machine_id = System.get_my_machine_id() return rmq_config.has_section(machine_id)
def _has_plugin(self): """ Checks whether this BackendType has a plugin installed """ try: return True if Configuration.get('ovs.plugins.backend.' + self.code) else False except: return False
def _get_memcache_nodes(): """ Get the memcache nodes """ memcache_ini = RawConfigParser() memcache_ini.read(os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg')) nodes = [node.strip() for node in memcache_ini.get('main', 'nodes').split(',')] return [memcache_ini.get(node, 'location') for node in nodes]
def clean_logs(): """ Cleans audit trail logs """ days = int(Configuration.get('ovs.core.audittrails.keep')) mark = time.time() - days * 24 * 60 * 60 for log in LogList.get_logs(): if log.time < mark: log.delete()
def test_single_node(self): base_port = Configuration.getInt('ovs.ports.arakoon') cluster = 'one' node = sorted(TestArakoonInstaller.nodes.keys())[0] ArakoonInstaller.create_cluster(cluster, node, []) contents = SSHClient.load(node).file_read(self._get_config_path(cluster)) expected = TestArakoonInstaller.expected_global.format(cluster, TestArakoonInstaller.nodes[node]) expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) self.assertEqual(contents.strip(), expected.strip())
def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._has_lock = False try: logger.debug('DS executing tick') self._mutex.acquire(wait=10) node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = Configuration.get('ovs.core.uniqueid') try: lock = self._persistent.get('{0}_lock'.format(self._namespace)) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen logger.debug('DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) logger.debug( 'DS stealing lock from {0}'.format(lock['name'])) self._load_schedule() self._has_lock = True else: logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} logger.debug('DS refreshing lock') self._persistent.set('{0}_lock'.format(self._namespace), lock) finally: self._mutex.release() if self._has_lock is True: logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due(entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval
def _get_memcache_nodes(): """ Get the memcache nodes """ memcache_ini = ConfigObj(os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg')) nodes = memcache_ini['main']['nodes'] if not isinstance(nodes, list): nodes = nodes.split(',') nodes = [node.strip() for node in nodes] return [memcache_ini[node]['location'] for node in nodes]
def __init__(self, vpool_name): self._vpool = vpool_name self._config_specfile = os.path.join( Configuration.get('ovs.core.cfgdir'), 'templates', 'volumedriverfs.json') if not os.path.exists( os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools')): os.makedirs( os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools')) self._config_file = os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools', '{}.json'.format(vpool_name)) self._config_tmpfile = os.path.join( Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools', '{}.json.tmp'.format(vpool_name)) self._config_readfile_handler = None self._config_file_handler = None self._config_specfile_handler = None self._config_file_content = None
def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._has_lock = False try: logger.debug("DS executing tick") self._mutex.acquire(wait=10) node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = Configuration.get("ovs.core.uniqueid") try: lock = self._persistent.get("{0}_lock".format(self._namespace)) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True logger.debug("DS there was no lock in tick") else: if lock["name"] == node_name: # The current node holds the lock logger.debug("DS keeps own lock") self._has_lock = True elif node_timestamp - lock["timestamp"] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen logger.debug("DS last lock refresh is {0}s old".format(node_timestamp - lock["timestamp"])) logger.debug("DS stealing lock from {0}".format(lock["name"])) self._load_schedule() self._has_lock = True else: logger.debug("DS lock is not ours") if self._has_lock is True: lock = {"name": node_name, "timestamp": node_timestamp} logger.debug("DS refreshing lock") self._persistent.set("{0}_lock".format(self._namespace), lock) finally: self._mutex.release() if self._has_lock is True: logger.debug("DS executing tick workload") remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due(entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass logger.debug("DS executing tick workload - done") return min(remaining_times + [self.max_interval]) else: return self.max_interval
def run_event_consumer(): """ Check whether to run the event consumer """ rmq_ini = ConfigObj( os.path.join(Configuration.get('ovs.core.cfgdir'), 'rabbitmqclient.cfg')) rmq_nodes = rmq_ini.get('main')['nodes'] if type( rmq_ini.get('main')['nodes']) == list else [ rmq_ini.get('main')['nodes'], ] machine_id = System.get_my_machine_id() return machine_id in rmq_nodes
def get_client(client_type=None): """ Returns a persistent storage client """ if not hasattr(PersistentFactory, 'store') or PersistentFactory.store is None: if client_type is None: client_type = Configuration.get('ovs.core.storage.persistent') PersistentFactory.store = None if client_type == 'arakoon': from ovs.extensions.storage.persistent.arakoonstore import ArakoonStore cluster = Configuration.get('ovs.core.db.arakoon.clusterid') PersistentFactory.store = ArakoonStore(cluster) if client_type == 'default': from ovs.extensions.storage.persistent.dummystore import DummyPersistentStore PersistentFactory.store = DummyPersistentStore() if PersistentFactory.store is None: raise RuntimeError('Invalid client_type specified') return PersistentFactory.store
def get_client(client_type=None): """ Returns a persistent storage client """ if not hasattr(PersistentFactory, "store") or PersistentFactory.store is None: if client_type is None: client_type = Configuration.get("ovs.core.storage.persistent") PersistentFactory.store = None if client_type == "arakoon": from ovs.extensions.storage.persistent.arakoonstore import ArakoonStore cluster = Configuration.get("ovs.core.db.arakoon.clusterid") PersistentFactory.store = ArakoonStore(cluster) if client_type == "default": from ovs.extensions.storage.persistent.dummystore import DummyPersistentStore PersistentFactory.store = DummyPersistentStore() if PersistentFactory.store is None: raise RuntimeError("Invalid client_type specified") return PersistentFactory.store
def mds_checkup(): """ Validates the current MDS setup/configuration and takes actions where required """ mds_dict = {} for vpool in VPoolList.get_vpools(): for mds_service in vpool.mds_services: storagerouter = mds_service.service.storagerouter if vpool not in mds_dict: mds_dict[vpool] = {} if storagerouter not in mds_dict[vpool]: mds_dict[vpool][storagerouter] = [] mds_dict[vpool][storagerouter].append(mds_service) for vpool in mds_dict: # 1. First, make sure there's at least one MDS on every StorageRouter that's not overloaded # If not, create an extra MDS for that StorageRouter for storagerouter in mds_dict[vpool]: mds_services = mds_dict[vpool][storagerouter] has_room = False for mds_service in mds_services: load, _ = MDSServiceController.get_mds_load(mds_service) if load < Configuration.getInt('ovs.storagedriver.mds.maxload'): has_room = True break if has_room is False: client = SSHClient.load(storagerouter.ip) mds_service = MDSServiceController.prepare_mds_service(client, storagerouter, vpool, fresh_only=False, start=True) if mds_service is None: raise RuntimeError('Could not add MDS node') mds_dict[vpool][storagerouter].append(mds_service) mds_config_set = MDSServiceController.get_mds_storagedriver_config_set(vpool) for storagerouter in mds_dict[vpool]: client = SSHClient.load(storagerouter.ip) storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name) storagedriver_config.load(client) if storagedriver_config.is_new is False: storagedriver_config.clean() # Clean out obsolete values storagedriver_config.configure_filesystem( fs_metadata_backend_mds_nodes=mds_config_set[storagerouter.guid] ) storagedriver_config.save(client) # 2. Per VPool, execute a safety check, making sure the master/slave configuration is optimal. for vdisk in vpool.vdisks: MDSServiceController.ensure_safety(vdisk)
def get_my_machine_id(client=None): """ Returns unique machine id based on mac address """ if not System.my_machine_id: ip_path = Configuration.get('ovs.core.ip.path') if ip_path is None: ip_path = "`which ip`" cmd = """{0} a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g' | sort""".format(ip_path) if client is None: output = check_output(cmd, shell=True).strip() else: output = client.run(cmd).strip() for mac in output.split('\n'): if mac.strip() != '000000000000': System.my_machine_id = mac.strip() break return System.my_machine_id
def collapse_arakoon(): logger.info('Starting arakoon collapse') arakoon_dir = os.path.join(Configuration.get('ovs.core.cfgdir'), 'arakoon') arakoon_clusters = map(lambda directory: os.path.basename(directory.rstrip(os.path.sep)), os.walk(arakoon_dir).next()[1]) for cluster in arakoon_clusters: logger.info(' Collapsing cluster: {}'.format(cluster)) cluster_instance = ArakoonManagementEx().getCluster(cluster) for node in cluster_instance.listNodes(): logger.info(' Collapsing node: {}'.format(node)) try: cluster_instance.remoteCollapse(node, 2) # Keep 2 tlogs except Exception as e: logger.info( 'Error during collapsing cluster {} node {}: {}\n{}'.format( cluster, node, str(e), traceback.format_exc() ) ) logger.info('Arakoon collapse finished')
def get_my_machine_id(client=None): """ Returns unique machine id based on mac address """ if not System.my_machine_id: ip_path = Configuration.get('ovs.core.ip.path') if ip_path is None: ip_path = "`which ip`" cmd = """{0} a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g' | sort""".format( ip_path) if client is None: output = check_output(cmd, shell=True).strip() else: output = client.run(cmd).strip() for mac in output.split('\n'): if mac.strip() != '000000000000': System.my_machine_id = mac.strip() break return System.my_machine_id
def test_multi_node(self): base_port = Configuration.getInt('ovs.ports.arakoon') cluster = 'one' nodes = sorted(TestArakoonInstaller.nodes.keys()) ArakoonInstaller.create_cluster(cluster, nodes[0], []) for node in nodes[1:]: ArakoonInstaller.extend_cluster(nodes[0], node, cluster, []) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes)) for node in nodes: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node in nodes: contents = SSHClient.load(node).file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip()) node = nodes[0] ArakoonInstaller.shrink_cluster(nodes[1], node, cluster) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:])) for node in nodes[1:]: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node in nodes[1:]: contents = SSHClient.load(node).file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip())
def services_running(target): try: key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) value = str(time.time()) if target == 'framework': # Volatile _log(target, 'Testing volatile store...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: from ovs.extensions.storage.volatilefactory import VolatileFactory VolatileFactory.store = None volatile = VolatileFactory.get_client() volatile.set(key, value) if volatile.get(key) == value: volatile.delete(key) break volatile.delete(key) except Exception as message: _log( target, ' Error during volatile store test: {0}'.format( message), 2) key = 'ovs-watcher-{0}'.format(str( uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: _log(target, ' Volatile store not working correctly', 2) return False _log(target, ' Volatile store OK after {0} tries'.format(tries), 0) # Persistent _log(target, 'Testing persistent store...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: from ovs.extensions.storage.persistentfactory import PersistentFactory PersistentFactory.store = None persistent = PersistentFactory.get_client() persistent.set(key, value) if persistent.get(key) == value: persistent.delete(key) break persistent.delete(key) except Exception as message: _log( target, ' Error during persistent store test: {0}'.format( message), 2) key = 'ovs-watcher-{0}'.format(str( uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: _log(target, ' Persistent store not working correctly', 2) return False _log(target, ' Persistent store OK after {0} tries'.format(tries), 0) if target == 'volumedriver': # Arakoon, voldrv cluster _log(target, 'Testing arakoon (voldrv)...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagementEx cluster = ArakoonManagementEx().getCluster('voldrv') client = cluster.getClient() client.set(key, value) if client.get(key) == value: client.delete(key) break client.delete(key) except Exception as message: _log( target, ' Error during arakoon (voldrv) test: {0}'.format( message), 2) key = 'ovs-watcher-{0}'.format(str( uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: _log(target, ' Arakoon (voldrv) not working correctly', 2) return False _log(target, ' Arakoon (voldrv) OK', 0) if target in ['framework', 'volumedriver']: # RabbitMQ _log(target, 'Test rabbitMQ...', 0) import pika from configobj import ConfigObj from ovs.plugin.provider.configuration import Configuration rmq_ini = ConfigObj( os.path.join(Configuration.get('ovs.core.cfgdir'), 'rabbitmqclient.cfg')) rmq_nodes = rmq_ini.get('main')['nodes'] if type( rmq_ini.get('main')['nodes']) == list else [ rmq_ini.get('main')['nodes'] ] rmq_servers = map(lambda m: rmq_ini.get(m)['location'], rmq_nodes) good_node = False for server in rmq_servers: try: connection_string = '{0}://{1}:{2}@{3}/%2F'.format( Configuration.get('ovs.core.broker.protocol'), Configuration.get('ovs.core.broker.login'), Configuration.get('ovs.core.broker.password'), server) connection = pika.BlockingConnection( pika.URLParameters(connection_string)) channel = connection.channel() channel.basic_publish( '', 'ovs-watcher', str(time.time()), pika.BasicProperties(content_type='text/plain', delivery_mode=1)) connection.close() good_node = True break except Exception as message: _log( target, ' Error during rabbitMQ test on node {0}: {1}'.format( server, message), 2) if good_node is False: _log(target, ' No working rabbitMQ node could be found', 2) return False _log(target, ' RabbitMQ test OK', 0) _log(target, 'All tests OK', 1) return True except Exception as ex: _log(target, 'Unexpected exception: {0}'.format(ex), 2) return False
for task in return_data['tasks']: self._process_task(task['task'], task['metadata']) except Exception, ex: logger.exception('Unexpected error processing tasks: {0}'.format(ex)) raise if 'interval' in return_data: interval = return_data['interval'] if interval != self.interval: self.interval = interval self._update_config('interval', str(interval)) self.interval = return_data['interval'] if __name__ == '__main__': try: if int(Configuration.get('ovs.support.enabled')) == 0: print 'Support not enabled' sys.exit(0) logger.info('Starting up') client = SupportAgent() while True: try: client.run() time.sleep(client.interval) except KeyboardInterrupt: raise except Exception, exception: logger.exception('Unexpected error during run: {0}'.format(exception)) time.sleep(10) except KeyboardInterrupt: print 'Aborting...'
and 'object' in [base.__name__ for base in member[1].__bases__]: this_mapping = member[1].mapping for key in this_mapping.keys(): if key not in mapping: mapping[key] = [] mapping[key] += this_mapping[key] logger.debug('Event map:') for key in mapping: logger.debug('{0}: {1}'.format(mapping[key][0]['property'], [ current_map['task'].__name__ for current_map in mapping[key] ])) # Starting connection and handling rmq_ini = ConfigObj( os.path.join(Configuration.get('ovs.core.cfgdir'), 'rabbitmqclient.cfg')) rmq_nodes = rmq_ini.get('main')['nodes'] if type( rmq_ini.get('main')['nodes']) == list else [ rmq_ini.get('main')['nodes'] ] this_server = '{0}:{1}'.format( Configuration.get('ovs.grid.ip'), Configuration.get('ovs.core.broker.port')) rmq_servers = [this_server] + [ server for server in map(lambda m: rmq_ini.get(m)['location'], rmq_nodes) if server != this_server ] channel = None server = '' loglevel = logging.root.manager.disable # Workaround for disabling logging
def __init__(self): self._config_corefile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-core.conf') self._config_exportfile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-export.conf')
def _get_int(key): return int(Configuration.get(key))
def ensure_safety(vdisk, excluded_storagerouters=None): """ Ensures (or tries to ensure) the safety of a given vdisk (except hypervisor). Assumptions: * A local overloaded master is better than a non-local non-overloaded master * Prefer master/services to be on different hosts, a subsequent slave on the same node doesn't add safety * Don't actively overload services (e.g. configure an MDS as slave causing it to get overloaded) * Too much safety is not wanted (it adds loads to nodes while not required) """ logger.debug('Ensuring MDS safety for vdisk {0}'.format(vdisk.guid)) vdisk.reload_client() if excluded_storagerouters is None: excluded_storagerouters = [] maxload = Configuration.getInt('ovs.storagedriver.mds.maxload') safety = Configuration.getInt('ovs.storagedriver.mds.safety') tlogs = Configuration.getInt('ovs.storagedriver.mds.tlogs') services = [mds_service.service for mds_service in vdisk.vpool.mds_services if mds_service.service.storagerouter not in excluded_storagerouters] nodes = set(service.storagerouter.ip for service in services) services_load = {} service_per_key = {} for service in services: load, load_plus = MDSServiceController.get_mds_load(service.mds_service) services_load[service.guid] = load, load_plus service_per_key['{0}:{1}'.format(service.storagerouter.ip, service.ports[0])] = service # List current configuration and filter out excluded services reconfigure_required = False reconfigure_reasons = [] vdisk.invalidate_dynamics(['info', 'storagedriver_id', 'storagerouter_guid']) configs = vdisk.info['metadata_backend_config'] for config in configs: config['key'] = '{0}:{1}'.format(config['ip'], config['port']) master_service = None if len(configs) > 0: config = configs[0] if config['key'] in service_per_key: master_service = service_per_key.get(config['key']) configs.remove(config) else: reconfigure_required = True reconfigure_reasons.append('Master ({0}:{1}) cannot be used anymore'.format(config['ip'], config['port'])) slave_services = [] for config in configs: if config['key'] in service_per_key: slave_services.append(service_per_key[config['key']]) else: reconfigure_required = True reconfigure_reasons.append('Slave ({0}:{1}) cannot be used anymore'.format(config['ip'], config['port'])) # Fix services_load services_per_load = {} for service in services: if service == master_service or service in slave_services: load = services_load[service.guid][0] else: load = services_load[service.guid][1] services_load[service.guid] = load if load not in services_per_load: services_per_load[load] = [] services_per_load[load].append(service) # Further checks if a reconfiguration is required. service_nodes = [] if master_service is not None: service_nodes.append(master_service.storagerouter.ip) for service in slave_services: ip = service.storagerouter.ip if ip in service_nodes: reconfigure_required = True reconfigure_reasons.append('Multiple MDS services on the same node') else: service_nodes.append(ip) if len(service_nodes) > safety: # Too much safety reconfigure_required = True reconfigure_reasons.append('Too much safety') if len(service_nodes) < safety and len(service_nodes) < len(nodes): # Insufficient MDS services configured while there should be sufficient nodes available reconfigure_required = True reconfigure_reasons.append('Not enough safety') if master_service is not None and services_load[master_service.guid] > maxload: # The master service is overloaded reconfigure_required = True reconfigure_reasons.append('Master overloaded') if master_service is not None and master_service.storagerouter_guid != vdisk.storagerouter_guid: # The master is not local reconfigure_required = True reconfigure_reasons.append('Master is not local') if any(service for service in slave_services if services_load[service.guid] > maxload): # There's a slave service overloaded reconfigure_required = True reconfigure_reasons.append('One or more slaves overloaded') if reconfigure_required is False: logger.debug('No reconfiguration required for vdisk {0}'.format(vdisk.guid)) MDSServiceController.sync_vdisk_to_reality(vdisk) return logger.debug('Reconfiguration required for vdisk {0}:'.format(vdisk.guid)) for reason in reconfigure_reasons: logger.debug('Reason: {0} - vdisk {1}'.format(reason, vdisk.guid)) # Prepare fresh configuration new_services = [] # Check whether the master (if available) is non-local to the vdisk and/or is overloaded master_ok = master_service is not None if master_ok is True: master_ok = master_service.storagerouter_guid == vdisk.storagerouter_guid and services_load[master_service.guid] <= maxload if master_ok: # Add this master to the fresh configuration new_services.append(master_service) else: # Try to find the best non-overloaded local MDS (slave) candidate_master = None candidate_master_load = 0 local_mds = None local_mds_load = 0 for service in services: load = services_load[service.guid] if load <= maxload and service.storagerouter_guid == vdisk.storagerouter_guid: if local_mds is None or local_mds_load > load: # This service is a non-overloaded local MDS local_mds = service local_mds_load = load if service in slave_services: if candidate_master is None or candidate_master_load > load: # This service is a non-overloaded local slave candidate_master = service candidate_master_load = load if candidate_master is not None: # A non-overloaded local slave was found. client = MetadataServerClient.load(candidate_master) amount_of_tlogs = client.catch_up(str(vdisk.volume_id), True) if amount_of_tlogs < tlogs: # Almost there. Catching up right now, and continue as soon as it's up-to-date start = time.time() client.catch_up(str(vdisk.volume_id), False) logger.debug('MDS catch up for vdisk {0} took {1}s'.format(vdisk.guid, round(time.time() - start, 2))) # It's up to date, so add it as a new master new_services.append(candidate_master) if master_service is not None: # The current master (if available) is now candidate for become one of the slaves slave_services.append(master_service) else: # It's not up to date, keep the previous master (if available) and give the local slave # some more time to catch up if master_service is not None: new_services.append(master_service) new_services.append(candidate_master) if candidate_master in slave_services: slave_services.remove(candidate_master) else: # There's no non-overloaded local slave found. Keep the current master (if available) and add # a local MDS (if available) as slave if master_service is not None: new_services.append(master_service) if local_mds is not None: new_services.append(local_mds) if local_mds in slave_services: slave_services.remove(local_mds) # At this point, there might (or might not) be a (new) master, and a (catching up) slave. The rest of the non-local # MDS nodes must now be added to the configuration until the safety is reached. There's always one extra # slave recycled to make sure there's always an (almost) up-to-date slave ready for failover loads = sorted(load for load in services_per_load.keys() if load <= maxload) nodes = set(service.storagerouter.ip for service in new_services) slave_added = False if len(nodes) < safety: for load in loads: for service in services_per_load[load]: if slave_added is False and service in slave_services and service.storagerouter.ip not in nodes: new_services.append(service) slave_services.remove(service) nodes.add(service.storagerouter.ip) slave_added = True if len(nodes) < safety: for load in loads: for service in services_per_load[load]: if len(nodes) < safety and service.storagerouter.ip not in nodes: new_services.append(service) nodes.add(service.storagerouter.ip) # Build the new configuration and update the vdisk configs = [] for service in new_services: client = MetadataServerClient.load(service) client.create_namespace(str(vdisk.volume_id)) configs.append(MDSNodeConfig(address=str(service.storagerouter.ip), port=service.ports[0])) vdisk.storagedriver_client.update_metadata_backend_config( volume_id=str(vdisk.volume_id), metadata_backend_config=MDSMetaDataBackendConfig(configs) ) MDSServiceController.sync_vdisk_to_reality(vdisk) logger.debug('Ensuring MDS safety for vdisk {0} completed'.format(vdisk.guid))
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time from subprocess import check_output from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.extensions.generic.system import System from ovs.plugin.provider.configuration import Configuration current_time = int(time.time()) machine_id = System.get_my_machine_id() amqp = '{0}://{1}:{2}@{3}//'.format(Configuration.get('ovs.core.broker.protocol'), Configuration.get('ovs.core.broker.login'), Configuration.get('ovs.core.broker.password'), Configuration.get('ovs.grid.ip')) celery_path = Configuration.get('ovs.core.celery.path') worker_states = check_output("{0} inspect ping -b {1} 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true".format(celery_path, amqp), shell=True) routers = StorageRouterList.get_storagerouters() for node in routers: if node.heartbeats is None: node.heartbeats = {} if 'celery@{0}: OK'.format(node.name) in worker_states: node.heartbeats['celery'] = current_time if node.machine_id == machine_id: node.heartbeats['process'] = current_time node.save()
def add_vpool(parameters): """ Add a vPool to the machine this task is running on """ parameters = {} if parameters is None else parameters ip = parameters['storagerouter_ip'] vpool_name = parameters['vpool_name'] if StorageRouterController._validate_ip(ip) is False: raise ValueError('The entered ip address is invalid') if not re.match('^[0-9a-z]+(\-+[0-9a-z]+)*$', vpool_name): raise ValueError('Invalid vpool_name given') client = SSHClient.load(ip) # Make sure to ALWAYS reload the client, as Fabric seems to be singleton-ish unique_id = System.get_my_machine_id(client) storagerouter = None for current_storagerouter in StorageRouterList.get_storagerouters(): if current_storagerouter.ip == ip and current_storagerouter.machine_id == unique_id: storagerouter = current_storagerouter break if storagerouter is None: raise RuntimeError('Could not find Storage Router with given ip address') vpool = VPoolList.get_vpool_by_name(vpool_name) storagedriver = None if vpool is not None: if vpool.backend_type.code == 'local': # Might be an issue, investigating whether it's on the same not or not if len(vpool.storagedrivers) == 1 and vpool.storagedrivers[0].storagerouter.machine_id != unique_id: raise RuntimeError('A local vPool with name {0} already exists'.format(vpool_name)) for vpool_storagedriver in vpool.storagedrivers: if vpool_storagedriver.storagerouter_guid == storagerouter.guid: storagedriver = vpool_storagedriver # The vPool is already added to this Storage Router and this might be a cleanup/recovery # Check whether there are running machines on this vPool machine_guids = [] for vdisk in vpool.vdisks: if vdisk.vmachine_guid not in machine_guids: machine_guids.append(vdisk.vmachine_guid) if vdisk.vmachine.hypervisor_status in ['RUNNING', 'PAUSED']: raise RuntimeError( 'At least one vMachine using this vPool is still running or paused. Make sure there are no active vMachines' ) nodes = {ip} # Set comprehension if vpool is not None: for vpool_storagedriver in vpool.storagedrivers: nodes.add(vpool_storagedriver.storagerouter.ip) nodes = list(nodes) services = ['volumedriver_{0}'.format(vpool_name), 'failovercache_{0}'.format(vpool_name)] # Stop services for node in nodes: node_client = SSHClient.load(node) for service in services: System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service if Service.has_service('{0}'): Service.disable_service('{0}') """.format(service)) System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service if Service.has_service('{0}'): Service.stop_service('{0}') """.format(service)) # Keep in mind that if the Storage Driver exists, the vPool does as well client = SSHClient.load(ip) mountpoint_bfs = '' directories_to_create = [] if vpool is None: vpool = VPool() supported_backends = System.read_remote_config(client, 'volumedriver.supported.backends').split(',') if 'rest' in supported_backends: supported_backends.remove('rest') # REST is not supported for now backend_type = BackendTypeList.get_backend_type_by_code(parameters['type']) vpool.backend_type = backend_type connection_host = connection_port = connection_username = connection_password = None if vpool.backend_type.code in ['local', 'distributed']: vpool.metadata = {'backend_type': 'LOCAL'} mountpoint_bfs = parameters['mountpoint_bfs'] directories_to_create.append(mountpoint_bfs) vpool.metadata['local_connection_path'] = mountpoint_bfs if vpool.backend_type.code == 'rest': connection_host = parameters['connection_host'] connection_port = parameters['connection_port'] rest_connection_timeout_secs = parameters['connection_timeout'] vpool.metadata = {'rest_connection_host': connection_host, 'rest_connection_port': connection_port, 'buchla_connection_log_level': "0", 'rest_connection_verbose_logging': rest_connection_timeout_secs, 'rest_connection_metadata_format': "JSON", 'backend_type': 'REST'} elif vpool.backend_type.code in ('ceph_s3', 'amazon_s3', 'swift_s3'): connection_host = parameters['connection_host'] connection_port = parameters['connection_port'] connection_username = parameters['connection_username'] connection_password = parameters['connection_password'] if vpool.backend_type.code in ['swift_s3']: strict_consistency = 'false' s3_connection_flavour = 'SWIFT' else: strict_consistency = 'true' s3_connection_flavour = 'S3' vpool.metadata = {'s3_connection_host': connection_host, 's3_connection_port': connection_port, 's3_connection_username': connection_username, 's3_connection_password': connection_password, 's3_connection_flavour': s3_connection_flavour, 's3_connection_strict_consistency': strict_consistency, 's3_connection_verbose_logging': 1, 'backend_type': 'S3'} vpool.name = vpool_name vpool.description = "{} {}".format(vpool.backend_type.code, vpool_name) vpool.login = connection_username vpool.password = connection_password if not connection_host: vpool.connection = None else: vpool.connection = '{}:{}'.format(connection_host, connection_port) vpool.save() # Connection information is Storage Driver related information new_storagedriver = False if storagedriver is None: storagedriver = StorageDriver() new_storagedriver = True mountpoint_temp = parameters['mountpoint_temp'] mountpoint_md = parameters['mountpoint_md'] mountpoint_readcache1 = parameters['mountpoint_readcache1'] mountpoint_readcache2 = parameters.get('mountpoint_readcache2', '') mountpoint_writecache = parameters['mountpoint_writecache'] mountpoint_foc = parameters['mountpoint_foc'] directories_to_create.append(mountpoint_temp) directories_to_create.append(mountpoint_md) directories_to_create.append(mountpoint_readcache1) if mountpoint_readcache2: directories_to_create.append(mountpoint_readcache2) directories_to_create.append(mountpoint_writecache) directories_to_create.append(mountpoint_foc) client = SSHClient.load(ip) dir_create_script = """ import os for directory in {0}: if not os.path.exists(directory): os.makedirs(directory) """.format(directories_to_create) System.exec_remote_python(client, dir_create_script) read_cache1_fs = os.statvfs(mountpoint_readcache1) read_cache2_fs = None if mountpoint_readcache2: read_cache2_fs = os.statvfs(mountpoint_readcache2) write_cache_fs = os.statvfs(mountpoint_writecache) fdcache = '{}/fd_{}'.format(mountpoint_writecache, vpool_name) scocache = '{}/sco_{}'.format(mountpoint_writecache, vpool_name) readcache1 = '{}/read1_{}'.format(mountpoint_readcache1, vpool_name) files2create = [readcache1] if mountpoint_readcache2 and mountpoint_readcache1 != mountpoint_readcache2: readcache2 = '{}/read2_{}'.format(mountpoint_readcache2, vpool_name) files2create.append(readcache2) else: readcache2 = '' failovercache = '{}/foc_{}'.format(mountpoint_foc, vpool_name) metadatapath = '{}/metadata_{}'.format(mountpoint_md, vpool_name) tlogpath = '{}/tlogs_{}'.format(mountpoint_md, vpool_name) rsppath = '/var/rsp/{}'.format(vpool_name) dirs2create = [scocache, failovercache, metadatapath, tlogpath, rsppath, System.read_remote_config(client, 'volumedriver.readcache.serialization.path')] cmd = "cat /etc/mtab | grep ^/dev/ | cut -d ' ' -f 2" mountpoints = [device.strip() for device in client.run(cmd).strip().split('\n')] mountpoints.remove('/') def is_partition(directory): for mountpoint in mountpoints: if directory == mountpoint: return True return False # Cache sizes # 20% = scocache # 20% = failovercache (@TODO: check if this can possibly consume more than 20%) # 60% = readcache # safety values: readcache1_factor = 0.2 readcache2_factor = 0.2 writecache_factor = 0.1 if (mountpoint_readcache1 == mountpoint_readcache2) or not mountpoint_readcache2: delta = set() delta.add(mountpoint_readcache1 if is_partition(mountpoint_readcache1) else '/dummy') delta.add(mountpoint_writecache if is_partition(mountpoint_writecache) else '/dummy') delta.add(mountpoint_foc if is_partition(mountpoint_foc) else '/dummy') if len(delta) == 1: readcache1_factor = 0.49 writecache_factor = 0.2 elif len(delta) == 2: if mountpoint_writecache == mountpoint_foc: readcache1_factor = 0.98 writecache_factor = 0.49 else: readcache1_factor = 0.49 if mountpoint_readcache1 == mountpoint_writecache: writecache_factor = 0.49 else: writecache_factor = 0.98 elif len(delta) == 3: readcache1_factor = 0.98 writecache_factor = 0.98 else: delta = set() delta.add(mountpoint_readcache1 if is_partition(mountpoint_readcache1) else '/dummy') delta.add(mountpoint_readcache2 if is_partition(mountpoint_readcache2) else '/dummy') delta.add(mountpoint_writecache if is_partition(mountpoint_writecache) else '/dummy') delta.add(mountpoint_foc if is_partition(mountpoint_foc) else '/dummy') if len(delta) == 1: # consider them all to be directories readcache1_factor = 0.24 readcache2_factor = 0.24 writecache_factor = 0.24 elif len(delta) == 2: if mountpoint_writecache == mountpoint_foc: writecache_factor = 0.24 if mountpoint_readcache1 == mountpoint_writecache: readcache1_factor = 0.49 readcache2_factor = 0.98 else: readcache1_factor = 0.98 readcache2_factor = 0.49 else: readcache1_factor = readcache2_factor = 0.49 writecache_factor = 0.49 elif len(delta) == 3: if mountpoint_writecache == mountpoint_foc: readcache1_factor = 0.98 readcache2_factor = 0.98 writecache_factor = 0.49 elif mountpoint_readcache1 == mountpoint_writecache: readcache1_factor = 0.49 readcache2_factor = 0.98 writecache_factor = 0.49 elif mountpoint_readcache1 == mountpoint_foc: readcache1_factor = 0.49 readcache2_factor = 0.98 writecache_factor = 0.98 elif mountpoint_readcache2 == mountpoint_writecache: readcache1_factor = 0.98 readcache2_factor = 0.49 writecache_factor = 0.49 elif mountpoint_readcache2 == mountpoint_foc: readcache1_factor = 0.98 readcache2_factor = 0.49 writecache_factor = 0.98 elif len(delta) == 4: readcache1_factor = 0.98 readcache2_factor = 0.98 writecache_factor = 0.98 # summarize caching on root partition (directory only) root_assigned = dict() if not is_partition(mountpoint_readcache1): root_assigned['readcache1_factor'] = readcache1_factor if not is_partition(mountpoint_readcache2): root_assigned['readcache2_factor'] = readcache2_factor if not is_partition(mountpoint_writecache): root_assigned['writecache_factor'] = writecache_factor if not is_partition(mountpoint_foc): root_assigned['foc_factor'] = min(readcache1_factor, readcache2_factor, writecache_factor) # always leave at least 20% of free space division_factor = 1.0 total_size = sum(root_assigned.values()) + .02 * len(root_assigned) if 0.8 < total_size < 1.6: division_factor = 2.0 elif 1.6 < total_size < 3.2: division_factor = 4.0 elif total_size >= 3.2: division_factor = 8.0 if 'readcache1_factor' in root_assigned.keys(): readcache1_factor /= division_factor if 'readcache2_factor' in root_assigned.keys(): readcache2_factor /= division_factor if 'writecache_factor' in root_assigned.keys(): writecache_factor /= division_factor scocache_size = '{0}KiB'.format((int(write_cache_fs.f_bavail * writecache_factor / 4096) * 4096) * 4) if (mountpoint_readcache1 and not mountpoint_readcache2) or (mountpoint_readcache1 == mountpoint_readcache2): mountpoint_readcache2 = '' readcache1_size = '{0}KiB'.format((int(read_cache1_fs.f_bavail * readcache1_factor / 4096) * 4096) * 4) readcache2 = '' readcache2_size = '0KiB' else: readcache1_size = '{0}KiB'.format((int(read_cache1_fs.f_bavail * readcache1_factor / 4096) * 4096) * 4) readcache2_size = '{0}KiB'.format((int(read_cache2_fs.f_bavail * readcache2_factor / 4096) * 4096) * 4) if new_storagedriver: ports_in_use = System.ports_in_use(client) ports_reserved = [] ports_in_use_model = {} for port_storagedriver in StorageDriverList.get_storagedrivers(): if port_storagedriver.vpool_guid not in ports_in_use_model: ports_in_use_model[port_storagedriver.vpool_guid] = port_storagedriver.ports ports_reserved += port_storagedriver.ports if vpool.guid in ports_in_use_model: # The vPool is extended to another StorageRouter. We need to use these ports. ports = ports_in_use_model[vpool.guid] if any(port in ports_in_use for port in ports): raise RuntimeError('The required ports are in use') else: # First StorageDriver for this vPool, so generating new ports ports = [] for port_range in System.read_remote_config(client, 'volumedriver.filesystem.ports').split(','): port_range = port_range.strip() if '-' in port_range: current_range = (int(port_range.split('-')[0]), int(port_range.split('-')[1])) else: current_range = (int(port_range), 65536) current_port = current_range[0] while len(ports) < 3: if current_port not in ports_in_use and current_port not in ports_reserved: ports.append(current_port) current_port += 1 if current_port > current_range[1]: break if len(ports) != 3: raise RuntimeError('Could not find enough free ports') else: ports = storagedriver.ports ip_path = Configuration.get('ovs.core.ip.path') if ip_path is None: ip_path = "`which ip`" cmd = "{0} a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1".format(ip_path) ipaddresses = client.run(cmd).strip().split('\n') ipaddresses = [ipaddr.strip() for ipaddr in ipaddresses] grid_ip = System.read_remote_config(client, 'ovs.grid.ip') if grid_ip in ipaddresses: ipaddresses.remove(grid_ip) if not ipaddresses: raise RuntimeError('No available ip addresses found suitable for Storage Router storage ip') if storagerouter.pmachine.hvtype == 'KVM': volumedriver_storageip = '127.0.0.1' else: volumedriver_storageip = parameters['storage_ip'] vrouter_id = '{0}{1}'.format(vpool_name, unique_id) vrouter_config = {'vrouter_id': vrouter_id, 'vrouter_redirect_timeout_ms': '5000', 'vrouter_routing_retries': 10, 'vrouter_volume_read_threshold': 1024, 'vrouter_volume_write_threshold': 1024, 'vrouter_file_read_threshold': 1024, 'vrouter_file_write_threshold': 1024, 'vrouter_min_workers': 4, 'vrouter_max_workers': 16} voldrv_arakoon_cluster_id = str(System.read_remote_config(client, 'volumedriver.arakoon.clusterid')) voldrv_arakoon_cluster = ArakoonManagementEx().getCluster(voldrv_arakoon_cluster_id) voldrv_arakoon_client_config = voldrv_arakoon_cluster.getClientConfig() arakoon_node_configs = [] for arakoon_node in voldrv_arakoon_client_config.keys(): arakoon_node_configs.append(ArakoonNodeConfig(arakoon_node, voldrv_arakoon_client_config[arakoon_node][0][0], voldrv_arakoon_client_config[arakoon_node][1])) vrouter_clusterregistry = ClusterRegistry(str(vpool.guid), voldrv_arakoon_cluster_id, arakoon_node_configs) node_configs = [] for existing_storagedriver in StorageDriverList.get_storagedrivers(): if existing_storagedriver.vpool_guid == vpool.guid: node_configs.append(ClusterNodeConfig(str(existing_storagedriver.storagedriver_id), str(existing_storagedriver.cluster_ip), existing_storagedriver.ports[0], existing_storagedriver.ports[1], existing_storagedriver.ports[2])) if new_storagedriver: node_configs.append(ClusterNodeConfig(vrouter_id, grid_ip, ports[0], ports[1], ports[2])) vrouter_clusterregistry.set_node_configs(node_configs) readcaches = [{'path': readcache1, 'size': readcache1_size}] if readcache2: readcaches.append({'path': readcache2, 'size': readcache2_size}) scocaches = [{'path': scocache, 'size': scocache_size}] filesystem_config = {'fs_backend_path': mountpoint_bfs} volumemanager_config = {'metadata_path': metadatapath, 'tlog_path': tlogpath} storagedriver_config_script = """ from ovs.plugin.provider.configuration import Configuration from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration fd_config = {{'fd_cache_path': '{11}', 'fd_extent_cache_capacity': '1024', 'fd_namespace' : 'fd-{0}-{12}'}} storagedriver_configuration = StorageDriverConfiguration('{0}') storagedriver_configuration.configure_backend({1}) storagedriver_configuration.configure_readcache({2}, Configuration.get('volumedriver.readcache.serialization.path') + '/{0}') storagedriver_configuration.configure_scocache({3}, '1GB', '2GB') storagedriver_configuration.configure_failovercache('{4}') storagedriver_configuration.configure_filesystem({5}) storagedriver_configuration.configure_volumemanager({6}) storagedriver_configuration.configure_volumerouter('{12}', {7}) storagedriver_configuration.configure_arakoon_cluster('{8}', {9}) storagedriver_configuration.configure_hypervisor('{10}') storagedriver_configuration.configure_filedriver(fd_config) """.format(vpool_name, vpool.metadata, readcaches, scocaches, failovercache, filesystem_config, volumemanager_config, vrouter_config, voldrv_arakoon_cluster_id, voldrv_arakoon_client_config, storagerouter.pmachine.hvtype, fdcache, vpool.guid) System.exec_remote_python(client, storagedriver_config_script) remote_script = """ import os from configobj import ConfigObj from ovs.plugin.provider.configuration import Configuration protocol = Configuration.get('ovs.core.broker.protocol') login = Configuration.get('ovs.core.broker.login') password = Configuration.get('ovs.core.broker.password') vpool_name = {0} uris = [] cfg = ConfigObj('/opt/OpenvStorage/config/rabbitmqclient.cfg') main_section = cfg.get('main') nodes = main_section['nodes'] if type(main_section['nodes']) == list else [main_section['nodes']] for node in nodes: uris.append({{'amqp_uri': '{{0}}://{{1}}:{{2}}@{{3}}'.format(protocol, login, password, cfg.get(node)['location'])}}) from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration queue_config = {{'events_amqp_routing_key': Configuration.get('ovs.core.broker.volumerouter.queue'), 'events_amqp_uris': uris}} for config_file in os.listdir('/opt/OpenvStorage/config/voldrv_vpools'): this_vpool_name = config_file.replace('.json', '') if config_file.endswith('.json') and (vpool_name is None or vpool_name == this_vpool_name): storagedriver_configuration = StorageDriverConfiguration(this_vpool_name) storagedriver_configuration.configure_event_publisher(queue_config) """.format(vpool_name if vpool_name is None else "'{0}'".format(vpool_name)) System.exec_remote_python(client, remote_script) # Updating the model storagedriver.storagedriver_id = vrouter_id storagedriver.name = vrouter_id.replace('_', ' ') storagedriver.description = storagedriver.name storagedriver.storage_ip = volumedriver_storageip storagedriver.cluster_ip = grid_ip storagedriver.ports = ports storagedriver.mountpoint = '/mnt/{0}'.format(vpool_name) storagedriver.mountpoint_temp = mountpoint_temp storagedriver.mountpoint_readcache1 = mountpoint_readcache1 storagedriver.mountpoint_readcache2 = mountpoint_readcache2 storagedriver.mountpoint_writecache = mountpoint_writecache storagedriver.mountpoint_foc = mountpoint_foc storagedriver.mountpoint_bfs = mountpoint_bfs storagedriver.mountpoint_md = mountpoint_md storagedriver.storagerouter = storagerouter storagedriver.vpool = vpool storagedriver.save() dirs2create.append(storagedriver.mountpoint) dirs2create.append(mountpoint_writecache + '/' + '/fd_' + vpool_name) dirs2create.append('{0}/fd_{1}'.format(mountpoint_writecache, vpool_name)) file_create_script = """ import os for directory in {0}: if not os.path.exists(directory): os.makedirs(directory) for filename in {1}: if not os.path.exists(filename): open(filename, 'a').close() """.format(dirs2create, files2create) System.exec_remote_python(client, file_create_script) voldrv_config_file = '{0}/voldrv_vpools/{1}.json'.format(System.read_remote_config(client, 'ovs.core.cfgdir'), vpool_name) log_file = '/var/log/ovs/volumedriver/{0}.log'.format(vpool_name) vd_cmd = '/usr/bin/volumedriver_fs -f --config-file={0} --mountpoint {1} --logrotation --logfile {2} -o big_writes -o sync_read -o allow_other'.format( voldrv_config_file, storagedriver.mountpoint, log_file) if storagerouter.pmachine.hvtype == 'KVM': vd_stopcmd = 'umount {0}'.format(storagedriver.mountpoint) else: vd_stopcmd = 'exportfs -u *:{0}; umount {0}'.format(storagedriver.mountpoint) vd_name = 'volumedriver_{}'.format(vpool_name) log_file = '/var/log/ovs/volumedriver/foc_{0}.log'.format(vpool_name) fc_cmd = '/usr/bin/failovercachehelper --config-file={0} --logfile={1}'.format(voldrv_config_file, log_file) fc_name = 'failovercache_{0}'.format(vpool_name) params = {'<VPOOL_MOUNTPOINT>': storagedriver.mountpoint, '<HYPERVISOR_TYPE>': storagerouter.pmachine.hvtype, '<VPOOL_NAME>': vpool_name, '<UUID>': str(uuid.uuid4())} if Osdist.is_ubuntu(client): if client.file_exists('/opt/OpenvStorage/config/templates/upstart/ovs-volumedriver.conf'): client.run('cp -f /opt/OpenvStorage/config/templates/upstart/ovs-volumedriver.conf /opt/OpenvStorage/config/templates/upstart/ovs-volumedriver_{0}.conf'.format(vpool_name)) client.run('cp -f /opt/OpenvStorage/config/templates/upstart/ovs-failovercache.conf /opt/OpenvStorage/config/templates/upstart/ovs-failovercache_{0}.conf'.format(vpool_name)) else: if client.file_exists('/opt/OpenvStorage/config/templates/systemd/ovs-volumedriver.service'): client.run('cp -f /opt/OpenvStorage/config/templates/systemd/ovs-volumedriver.service /opt/OpenvStorage/config/templates/systemd/ovs-volumedriver_{0}.service'.format(vpool_name)) client.run('cp -f /opt/OpenvStorage/config/templates/systemd/ovs-failovercache.service /opt/OpenvStorage/config/templates/systemd/ovs-failovercache_{0}.service'.format(vpool_name)) service_script = """ from ovs.plugin.provider.service import Service Service.add_service(package=('openvstorage', 'volumedriver'), name='{0}', command='{1}', stop_command='{2}', params={5}) Service.add_service(package=('openvstorage', 'failovercache'), name='{3}', command='{4}', stop_command=None, params={5}) """.format( vd_name, vd_cmd, vd_stopcmd, fc_name, fc_cmd, params ) System.exec_remote_python(client, service_script) if storagerouter.pmachine.hvtype == 'VMWARE': client.run("grep -q '/tmp localhost(ro,no_subtree_check)' /etc/exports || echo '/tmp localhost(ro,no_subtree_check)' >> /etc/exports") if Osdist.is_ubuntu(client): client.run('service nfs-kernel-server start') else: client.run('service nfs start') if storagerouter.pmachine.hvtype == 'KVM': client.run('virsh pool-define-as {0} dir - - - - {1}'.format(vpool_name, storagedriver.mountpoint)) client.run('virsh pool-build {0}'.format(vpool_name)) client.run('virsh pool-start {0}'.format(vpool_name)) client.run('virsh pool-autostart {0}'.format(vpool_name)) # Start services for node in nodes: node_client = SSHClient.load(node) for service in services: System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service Service.enable_service('{0}') """.format(service)) System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service Service.start_service('{0}') """.format(service)) # Fill vPool size vfs_info = os.statvfs('/mnt/{0}'.format(vpool_name)) vpool.size = vfs_info.f_blocks * vfs_info.f_bsize vpool.save() # Configure Cinder ovsdb = PersistentFactory.get_client() vpool_config_key = str('ovs_openstack_cinder_%s' % storagedriver.vpool_guid) if ovsdb.exists(vpool_config_key): # Second node gets values saved by first node cinder_password, cinder_user, tenant_name, controller_ip, config_cinder = ovsdb.get(vpool_config_key) else: config_cinder = parameters.get('config_cinder', False) cinder_password = '' cinder_user = '' tenant_name = '' controller_ip = '' if config_cinder: cinder_password = parameters.get('cinder_pass', cinder_password) cinder_user = parameters.get('cinder_user', cinder_user) tenant_name = parameters.get('cinder_tenant', tenant_name) controller_ip = parameters.get('cinder_controller', controller_ip) # Keystone host if cinder_password: osc = OpenStackCinder(cinder_password = cinder_password, cinder_user = cinder_user, tenant_name = tenant_name, controller_ip = controller_ip) osc.configure_vpool(vpool_name, storagedriver.mountpoint) # Save values for first node to use ovsdb.set(vpool_config_key, [cinder_password, cinder_user, tenant_name, controller_ip, config_cinder])
def process(queue, body, mapping): """ Processes the actual received body """ if queue == Configuration.get('ovs.core.broker.volumerouter.queue'): import json cache = VolatileFactory.get_client() all_extensions = None message = FileSystemEvents.EventMessage() message.ParseFromString(body) # Possible special tags used as `arguments` key: # - [NODE_ID]: Replaced by the storagedriver_id as reported by the event # - [CLUSTER_ID]: Replaced by the clusterid as reported by the event # Possible deduping key tags: # - [EVENT_NAME]: The name of the eventmessage type # - [TASK_NAME]: Task method name # - [<argument value>]: Any value of the `arguments` dictionary. logger.info('Got event, processing...') event = None for extension in mapping.keys(): if not message.event.HasExtension(extension): continue event = message.event.Extensions[extension] node_id = message.node_id cluster_id = message.cluster_id for current_map in mapping[extension]: task = current_map['task'] kwargs = {} delay = 0 routing_key = 'generic' for field, target in current_map['arguments'].iteritems(): if field == '[NODE_ID]': kwargs[target] = node_id elif field == '[CLUSTER_ID]': kwargs[target] = cluster_id else: kwargs[target] = getattr(event, field) if 'options' in current_map: options = current_map['options'] if options.get('execonstoragerouter', False): storagedriver = StorageDriverList.get_by_storagedriver_id(node_id) if storagedriver is not None: routing_key = 'sr.{0}'.format(storagedriver.storagerouter.machine_id) delay = options.get('delay', 0) dedupe = options.get('dedupe', False) dedupe_key = options.get('dedupe_key', None) if dedupe is True and dedupe_key is not None: # We can't dedupe without a key key = dedupe_key key = key.replace('[EVENT_NAME]', extension.full_name) key = key.replace('[TASK_NAME]', task.__class__.__name__) for kwarg_key in kwargs: key = key.replace('[{0}]'.format(kwarg_key), kwargs[kwarg_key]) key = key.replace(' ', '_') task_id = cache.get(key) if task_id: # Key exists, task was already scheduled # If task is already running, the revoke message will # be ignored revoke(task_id) _log(task, kwargs, node_id) async_result = task.s(**kwargs).apply_async( countdown=delay, routing_key=routing_key ) cache.set(key, async_result.id, 600) # Store the task id new_task_id = async_result.id else: _log(task, kwargs, node_id) async_result = task.s(**kwargs).apply_async( countdown=delay, routing_key=routing_key ) new_task_id = async_result.id else: async_result = task.delay(**kwargs) new_task_id = async_result.id logger.info('[{0}] {1}({2}) started on {3} with taskid {4}. Delay: {5}s'.format( queue, task.__name__, json.dumps(kwargs), routing_key, new_task_id, delay )) if event is None: message_type = 'unknown' if all_extensions is None: all_extensions = _load_extensions() for extension in all_extensions: if message.event.HasExtension(extension): message_type = extension.full_name logger.info('A message with type {0} was received. Skipped.'.format(message_type)) else: raise NotImplementedError('Queue {} is not yet implemented'.format(queue))
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Django settings module """ import os from ovs.plugin.provider.configuration import Configuration DEBUG = False TEMPLATE_DEBUG = DEBUG UNIQUE_ID = Configuration.get('ovs.core.uniqueid') UI_NAME = Configuration.get('ovs.webapps.main.uiname') APP_NAME = Configuration.get('ovs.webapps.main.appname') BASE_WWW_DIR = os.path.dirname(__file__) BASE_FOLDER = os.path.join(Configuration.get('ovs.core.basedir'), Configuration.get('ovs.webapps.dir'), APP_NAME) VERSION = (1,) # This tuple should contain all supported versions. E.g.: (1,) or (1, 2) or (1, 2, 3) or (2, 3, 4) or ... BASE_LOG_DIR = '/var/log/ovs' LOG_FILENAME = '/django.log' FRONTEND_ROOT = '/' + UI_NAME STATIC_URL = '/' + UI_NAME + '/static/' # STATIC_URL must end with a slash FORCE_SCRIPT_NAME = FRONTEND_ROOT
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Django settings module """ import os from ovs.plugin.provider.configuration import Configuration DEBUG = False TEMPLATE_DEBUG = DEBUG UI_NAME = Configuration.get('ovs.webapps.main.uiname') APP_NAME = Configuration.get('ovs.webapps.main.appname') BASE_WWW_DIR = os.path.dirname(__file__) BASE_FOLDER = os.path.join(Configuration.get('ovs.core.basedir'), Configuration.get('ovs.webapps.dir'), APP_NAME) VERSION = ( 1, ) # This tuple should contain all supported versions. E.g.: (1,) or (1, 2) or (1, 2, 3) or (2, 3, 4) or ... BASE_LOG_DIR = '/var/log/ovs' LOG_FILENAME = '/django.log' FRONTEND_ROOT = '/' + UI_NAME STATIC_URL = '/' + UI_NAME + '/static/' # STATIC_URL must end with a slash
def services_running(target): try: key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) value = str(time.time()) if target == 'framework': # Volatile _log(target, 'Testing volatile store...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: from ovs.extensions.storage.volatilefactory import VolatileFactory VolatileFactory.store = None volatile = VolatileFactory.get_client() volatile.set(key, value) if volatile.get(key) == value: volatile.delete(key) break volatile.delete(key) except Exception as message: _log(target, ' Error during volatile store test: {0}'.format(message), 2) key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: _log(target, ' Volatile store not working correctly', 2) return False _log(target, ' Volatile store OK after {0} tries'.format(tries), 0) # Persistent _log(target, 'Testing persistent store...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: from ovs.extensions.storage.persistentfactory import PersistentFactory PersistentFactory.store = None persistent = PersistentFactory.get_client() persistent.set(key, value) if persistent.get(key) == value: persistent.delete(key) break persistent.delete(key) except Exception as message: _log(target, ' Error during persistent store test: {0}'.format(message), 2) key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: _log(target, ' Persistent store not working correctly', 2) return False _log(target, ' Persistent store OK after {0} tries'.format(tries), 0) if target == 'volumedriver': # Arakoon, voldrv cluster _log(target, 'Testing arakoon (voldrv)...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagementEx cluster = ArakoonManagementEx().getCluster('voldrv') client = cluster.getClient() client.set(key, value) if client.get(key) == value: client.delete(key) break client.delete(key) except Exception as message: _log(target, ' Error during arakoon (voldrv) test: {0}'.format(message), 2) key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: _log(target, ' Arakoon (voldrv) not working correctly', 2) return False _log(target, ' Arakoon (voldrv) OK', 0) if target in ['framework', 'volumedriver']: # RabbitMQ _log(target, 'Test rabbitMQ...', 0) import pika from ConfigParser import RawConfigParser from ovs.plugin.provider.configuration import Configuration rmq_ini = RawConfigParser() rmq_ini.read(os.path.join(Configuration.get('ovs.core.cfgdir'), 'rabbitmqclient.cfg')) rmq_nodes = [node.strip() for node in rmq_ini.get('main', 'nodes').split(',')] rmq_servers = map(lambda n: rmq_ini.get(n, 'location'), rmq_nodes) good_node = False for server in rmq_servers: try: connection_string = '{0}://{1}:{2}@{3}/%2F'.format(Configuration.get('ovs.core.broker.protocol'), Configuration.get('ovs.core.broker.login'), Configuration.get('ovs.core.broker.password'), server) connection = pika.BlockingConnection(pika.URLParameters(connection_string)) channel = connection.channel() channel.basic_publish('', 'ovs-watcher', str(time.time()), pika.BasicProperties(content_type='text/plain', delivery_mode=1)) connection.close() good_node = True break except Exception as message: _log(target, ' Error during rabbitMQ test on node {0}: {1}'.format(server, message), 2) if good_node is False: _log(target, ' No working rabbitMQ node could be found', 2) return False _log(target, ' RabbitMQ test OK', 0) _log(target, 'All tests OK', 1) return True except Exception as ex: _log(target, 'Unexpected exception: {0}'.format(ex), 2) return False
def process(queue, body, mapping): """ Processes the actual received body """ if queue == Configuration.get('ovs.core.broker.volumerouter.queue'): import json import volumedriver.storagerouter.EventMessages_pb2 as EventMessages cache = VolatileFactory.get_client() data = EventMessages.EventMessage().FromString(body) # Possible special tags used as `arguments` key: # - [NODE_ID]: Replaced by the storagedriver_id as reported by the event # - [CLUSTER_ID]: Replaced by the clusterid as reported by the event # Possible deduping key tags: # - [EVENT_NAME]: The name of the eventmessage type # - [TASK_NAME]: Task method name # - [<argument value>]: Any value of the `arguments` dictionary. if data.type in mapping: for current_map in mapping[data.type]: task = current_map['task'] data_container = getattr(data, current_map['property']) kwargs = {} delay = 0 routing_key = 'generic' for field, target in current_map['arguments'].iteritems(): if field == '[NODE_ID]': kwargs[target] = data.node_id elif field == '[CLUSTER_ID]': kwargs[target] = data.cluster_id else: kwargs[target] = getattr(data_container, field) if 'options' in current_map: options = current_map['options'] if options.get('execonstoragerouter', False): storagedriver = StorageDriverList.get_by_storagedriver_id( data.node_id) if storagedriver is not None: routing_key = 'sr.{0}'.format( storagedriver.storagerouter.machine_id) delay = options.get('delay', 0) dedupe = options.get('dedupe', False) dedupe_key = options.get('dedupe_key', None) if dedupe is True and dedupe_key is not None: # We can't dedupe without a key key = dedupe_key key = key.replace('[EVENT_NAME]', data.type.__class__.__name__) key = key.replace('[TASK_NAME]', task.__class__.__name__) for kwarg_key in kwargs: key = key.replace('[{0}]'.format(kwarg_key), kwargs[kwarg_key]) key = key.replace(' ', '_') task_id = cache.get(key) if task_id: # Key exists, task was already scheduled # If task is already running, the revoke message will # be ignored revoke(task_id) _log(task, kwargs, data.node_id) async_result = task.s(**kwargs).apply_async( countdown=delay, routing_key=routing_key) cache.set(key, async_result.id, 600) # Store the task id new_task_id = async_result.id else: _log(task, kwargs, data.node_id) async_result = task.s(**kwargs).apply_async( countdown=delay, routing_key=routing_key) new_task_id = async_result.id else: async_result = task.delay(**kwargs) new_task_id = async_result.id logger.info( '[{0}] {1}({2}) started on {3} with taskid {4}. Delay: {5}s' .format(queue, task.__name__, json.dumps(kwargs), routing_key, new_task_id, delay)) else: logger.info('Message type {0} was received. Skipped.'.format( str(data.type))) else: raise NotImplementedError( 'Queue {} is not yet implemented'.format(queue))
def prepare_mds_service(client, storagerouter, vpool, fresh_only=True, start=False): """ Prepares an MDS service: * Creates the required configuration * Sets up the service files Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise configuration regarding both is completed. """ mdsservice_type = ServiceTypeList.get_by_name('MetadataServer') storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid][0] # Fetch service sequence number service_number = -1 for mds_service in vpool.mds_services: if mds_service.service.storagerouter_guid == storagerouter.guid: service_number = max(mds_service.number, service_number) if fresh_only is True and service_number >= 0: return None # There are already one or more MDS services running, aborting service_number += 1 # Find free port occupied_ports = [] for service in mdsservice_type.services: if service.storagerouter_guid == storagerouter.guid: occupied_ports.append(service.ports[0]) port = System.get_free_ports(Configuration.get('ovs.ports.mds'), exclude=occupied_ports, nr=1, client=client)[0] # Add service to the model service = Service() service.name = 'metadataserver_{0}_{1}'.format(vpool.name, service_number) service.type = mdsservice_type service.storagerouter = storagerouter service.ports = [port] service.save() mds_service = MDSService() mds_service.service = service mds_service.vpool = vpool mds_service.number = service_number mds_service.save() # Prepare some directores scratch_dir = '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_temp, vpool.name, service_number) rocksdb_dir = '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_md, vpool.name, service_number) client.run('mkdir -p {0}'.format(scratch_dir)) client.run('mkdir -p {0}'.format(rocksdb_dir)) # Generate the configuration file metadataserver_config = StorageDriverConfiguration('metadataserver', vpool.name, number=service_number) metadataserver_config.load(client) metadataserver_config.clean() # Clean out obsolete values if vpool.backend_type.code == 'alba': metadataserver_config.configure_backend_connection_manager(alba_connection_host='127.0.0.1', alba_connection_port=storagedriver.alba_proxy.service.ports[0], backend_type='ALBA') else: metadataserver_config.configure_backend_connection_manager(**vpool.metadata) metadataserver_config.configure_metadata_server(mds_address=storagerouter.ip, mds_port=service.ports[0], mds_scratch_dir=scratch_dir, mds_rocksdb_path=rocksdb_dir) metadataserver_config.save(client) # Create system services params = {'<VPOOL_NAME>': vpool.name, '<SERVICE_NUMBER>': str(service_number)} template_dir = '/opt/OpenvStorage/config/templates/upstart' client.run('cp -f {0}/ovs-metadataserver.conf {0}/ovs-metadataserver_{1}_{2}.conf'.format(template_dir, vpool.name, service_number)) service_script = """ from ovs.plugin.provider.service import Service Service.add_service(package=('openvstorage', 'metadataserver'), name='metadataserver_{0}_{1}', command=None, stop_command=None, params={2}) """.format(vpool.name, service_number, params) System.exec_remote_python(client, service_script) if start is True: System.exec_remote_python(client, """ from ovs.plugin.provider.service import Service Service.enable_service('{0}') """.format(service.name)) System.exec_remote_python(client, """ from ovs.plugin.provider.service import Service Service.start_service('{0}') """.format(service.name)) return mds_service
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time from subprocess import check_output from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.extensions.generic.system import System from ovs.plugin.provider.configuration import Configuration current_time = int(time.time()) machine_id = System.get_my_machine_id() amqp = '{0}://{1}:{2}@{3}//'.format( Configuration.get('ovs.core.broker.protocol'), Configuration.get('ovs.core.broker.login'), Configuration.get('ovs.core.broker.password'), Configuration.get('ovs.grid.ip')) celery_path = Configuration.get('ovs.core.celery.path') worker_states = check_output( "{0} inspect ping -b {1} 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true" .format(celery_path, amqp), shell=True) routers = StorageRouterList.get_storagerouters() for node in routers: if node.heartbeats is None: node.heartbeats = {} if 'celery@{0}: OK'.format(node.name) in worker_states: node.heartbeats['celery'] = current_time
import os from kombu import Queue from celery import Celery from celery.schedules import crontab from celery.signals import task_postrun, worker_process_init from ovs.lib.messaging import MessageController from ovs.log.logHandler import LogHandler from ovs.extensions.storage.volatilefactory import VolatileFactory from ovs.extensions.storage.persistentfactory import PersistentFactory from ovs.extensions.generic.system import System from ovs.plugin.provider.configuration import Configuration from configobj import ConfigObj memcache_ini = ConfigObj( os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg')) memcache_nodes = memcache_ini.get('main')['nodes'] if type( memcache_ini.get('main')['nodes']) == list else [ memcache_ini.get('main')['nodes'], ] memcache_servers = map(lambda m: memcache_ini.get(m)['location'], memcache_nodes) rmq_ini = ConfigObj( os.path.join(Configuration.get('ovs.core.cfgdir'), 'rabbitmqclient.cfg')) rmq_nodes = rmq_ini.get('main')['nodes'] if type( rmq_ini.get('main')['nodes']) == list else [ rmq_ini.get('main')['nodes'], ] rmq_servers = map(lambda m: rmq_ini.get(m)['location'], rmq_nodes)