if run_event_consumer is True: # Load mapping mapping = {} path = '/'.join([os.path.dirname(__file__), 'mappings']) for filename in os.listdir(path): if os.path.isfile('/'.join([path, filename])) and filename.endswith('.py'): name = filename.replace('.py', '') mod = imp.load_source(name, '/'.join([path, filename])) for member in inspect.getmembers(mod, predicate=inspect.isclass): if member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]: this_mapping = member[1].mapping for key in this_mapping.keys(): if key not in mapping: mapping[key] = [] mapping[key] += this_mapping[key] logger.debug('Event map:') for key in mapping: logger.debug('{0}: {1}'.format(key.name, [current_map['task'].__name__ for current_map in mapping[key]])) # Starting connection and handling rmq_servers = Configuration.get('/ovs/framework/messagequeue|endpoints') channel = None server = '' loglevel = logging.root.manager.disable # Workaround for disabling logging logging.disable('WARNING') for server in rmq_servers: try: connection = pika.BlockingConnection( pika.ConnectionParameters(host=server.split(':')[0], port=int(server.split(':')[1]), credentials=pika.PlainCredentials(
class StorageDriverConfiguration(object): """ StorageDriver configuration class """ CACHE_BLOCK = 'block_cache' CACHE_FRAGMENT = 'fragment_cache' def __init__(self, vpool_guid, storagedriver_id): """ Initializes the class """ _log_level = LOG_LEVEL_MAPPING[OVSLogger( 'extensions').getEffectiveLevel()] # noinspection PyCallByClass,PyTypeChecker storagerouterclient.Logger.setupLogging( OVSLogger.load_path('storagerouterclient'), _log_level) # noinspection PyArgumentList storagerouterclient.Logger.enableLogging() self._key = '/ovs/vpools/{0}/hosts/{1}/config'.format( vpool_guid, storagedriver_id) self._logger = OVSLogger('extensions') self._dirty_entries = [] self.remote_path = Configuration.get_configuration_path( self._key).strip('/') # Load configuration if Configuration.exists(self._key): self.configuration = Configuration.get(self._key) self.config_missing = False else: self.configuration = {} self.config_missing = True self._logger.debug( 'Could not find config {0}, a new one will be created'.format( self._key)) def save(self, client=None, force_reload=False): """ Saves the configuration to a given file, optionally a remote one :param client: If provided, save remote configuration :type client: ovs_extensions.generic.sshclient.SSHClient :param force_reload: Make sure the 'update_configuration' gets triggered. Should be used when configuration changes have been applied from 'outside' :type force_reload: bool :return: Changes to the configuration :rtype: list """ changes = [] Configuration.set(self._key, self.configuration) # No changes detected in the configuration management if len(self._dirty_entries) == 0 and force_reload is False: self._logger.debug('No need to apply changes, nothing changed') self.config_missing = False return changes # Retrieve the changes from volumedriver self._logger.info( 'Applying local storagedriver configuration changes{0}'.format( '' if client is None else ' on {0}'.format(client.ip))) reloaded = False try: if client is None: changes = LocalStorageRouterClient( self.remote_path).update_configuration(self.remote_path) else: with remote(client.ip, [LocalStorageRouterClient]) as rem: changes = copy.deepcopy( rem.LocalStorageRouterClient( self.remote_path).update_configuration( self.remote_path)) reloaded = True except Exception as ex: if 'ClusterNotReachableException' not in str(ex): raise # No changes if len(changes) == 0: if reloaded is True: if len(self._dirty_entries) > 0: self._logger.warning( 'Following changes were not applied: {0}'.format( ', '.join(self._dirty_entries))) else: self._logger.warning( 'Changes were not applied since StorageDriver is unavailable' ) self.config_missing = False self._dirty_entries = [] return changes # Verify the output of the changes and log them for change in changes: if not isinstance(change, dict): raise RuntimeError('Unexpected update_configuration output') if 'param_name' not in change or 'old_value' not in change or 'new_value' not in change: raise RuntimeError( 'Unexpected update_configuration output. Expected different keys, but got {0}' .format(', '.join(change.keys()))) param_name = change['param_name'] if force_reload is False: if param_name not in self._dirty_entries: raise RuntimeError( 'Unexpected configuration change: {0}'.format( param_name)) self._dirty_entries.remove(param_name) self._logger.info('Changed {0} from "{1}" to "{2}"'.format( param_name, change['old_value'], change['new_value'])) self._logger.info('Changes applied') if len(self._dirty_entries) > 0: self._logger.warning( 'Following changes were not applied: {0}'.format(', '.join( self._dirty_entries))) self.config_missing = False self._dirty_entries = [] return changes def __getattr__(self, item): from ovs_extensions.generic.toolbox import ExtensionsToolbox if item.startswith('configure_'): section = ExtensionsToolbox.remove_prefix(item, 'configure_') return lambda **kwargs: self._add(section, **kwargs) if item.startswith('clear_'): section = ExtensionsToolbox.remove_prefix(item, 'clear_') return lambda: self._delete(section) def _add(self, section, **kwargs): """ Configures a section """ for item, value in kwargs.iteritems(): if section not in self.configuration: self.configuration[section] = {} if item not in self.configuration[ section] or self.configuration[section][item] != value: self._dirty_entries.append(item) self.configuration[section][item] = value def _delete(self, section): """ Removes a section from the configuration """ if section in self.configuration: del self.configuration[section]
class Watcher(object): """ Watcher class """ LOG_CONTENTS = None def __init__(self): """ Dummy init method """ self._logger = Logger('extensions-generic') def log_message(self, log_target, entry, level): """ Logs an entry """ if level > 0: # 0 = debug, 1 = info, 2 = error self._logger.debug('[{0}] {1}'.format(log_target, entry)) def services_running(self, target): """ Check all services are running :param target: Target to check :return: Boolean """ try: key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) value = str(time.time()) if target in ['config', 'framework']: self.log_message(target, 'Testing configuration store...', 0) from ovs.extensions.generic.configuration import Configuration try: Configuration.list('/') except Exception as ex: self.log_message( target, ' Error during configuration store test: {0}'.format( ex), 2) return False from ovs.extensions.db.arakooninstaller import ArakoonInstaller, ArakoonClusterConfig from ovs_extensions.db.arakoon.pyrakoon.pyrakoon.compat import NoGuarantee from ovs.extensions.generic.configuration import Configuration with open(Configuration.CACC_LOCATION) as config_file: contents = config_file.read() config = ArakoonClusterConfig( cluster_id=Configuration.ARAKOON_NAME, load_config=False) config.read_config(contents=contents) client = ArakoonInstaller.build_client(config) contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY, consistency=NoGuarantee()) if Watcher.LOG_CONTENTS != contents: try: config.read_config( contents=contents ) # Validate whether the contents are not corrupt except Exception as ex: self.log_message( target, ' Configuration stored in configuration store seems to be corrupt: {0}' .format(ex), 2) return False temp_filename = '{0}~'.format(Configuration.CACC_LOCATION) with open(temp_filename, 'w') as config_file: config_file.write(contents) config_file.flush() os.fsync(config_file) os.rename(temp_filename, Configuration.CACC_LOCATION) Watcher.LOG_CONTENTS = contents self.log_message(target, ' Configuration store OK', 0) if target == 'framework': # Volatile self.log_message(target, 'Testing volatile store...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: try: logging.disable(logging.WARNING) from ovs.extensions.storage.volatilefactory import VolatileFactory VolatileFactory.store = None volatile = VolatileFactory.get_client() volatile.set(key, value) if volatile.get(key) == value: volatile.delete(key) break volatile.delete(key) finally: logging.disable(logging.NOTSET) except Exception as message: self.log_message( target, ' Error during volatile store test: {0}'.format( message), 2) key = 'ovs-watcher-{0}'.format(str( uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: self.log_message(target, ' Volatile store not working correctly', 2) return False self.log_message( target, ' Volatile store OK after {0} tries'.format(tries), 0) # Persistent self.log_message(target, 'Testing persistent store...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: try: logging.disable(logging.WARNING) persistent = PersistentFactory.get_client() persistent.nop() break finally: logging.disable(logging.NOTSET) except Exception as message: self.log_message( target, ' Error during persistent store test: {0}'.format( message), 2) time.sleep(1) tries += 1 if tries == max_tries: self.log_message( target, ' Persistent store not working correctly', 2) return False self.log_message( target, ' Persistent store OK after {0} tries'.format(tries), 0) if target == 'volumedriver': # Arakoon, voldrv cluster self.log_message(target, 'Testing arakoon (voldrv)...', 0) max_tries = 5 tries = 0 while tries < max_tries: try: from ovs.extensions.generic.configuration import Configuration from ovs_extensions.storage.persistent.pyrakoonstore import PyrakoonStore cluster_name = str( Configuration.get( '/ovs/framework/arakoon_clusters|voldrv')) configuration = Configuration.get( '/ovs/arakoon/{0}/config'.format(cluster_name), raw=True) client = PyrakoonStore(cluster=cluster_name, configuration=configuration) client.nop() break except Exception as message: self.log_message( target, ' Error during arakoon (voldrv) test: {0}'.format( message), 2) time.sleep(1) tries += 1 if tries == max_tries: self.log_message( target, ' Arakoon (voldrv) not working correctly', 2) return False self.log_message(target, ' Arakoon (voldrv) OK', 0) if target in ['framework', 'volumedriver']: # RabbitMQ self.log_message(target, 'Test rabbitMQ...', 0) import pika from ovs.extensions.generic.configuration import Configuration messagequeue = Configuration.get('/ovs/framework/messagequeue') rmq_servers = messagequeue['endpoints'] good_node = False for server in rmq_servers: try: connection_string = '{0}://{1}:{2}@{3}/%2F'.format( messagequeue['protocol'], messagequeue['user'], messagequeue['password'], server) connection = pika.BlockingConnection( pika.URLParameters(connection_string)) channel = connection.channel() channel.basic_publish( '', 'ovs-watcher', str(time.time()), pika.BasicProperties(content_type='text/plain', delivery_mode=1)) connection.close() good_node = True except Exception as message: self.log_message( target, ' Error during rabbitMQ test on node {0}: {1}'. format(server, message), 2) if good_node is False: self.log_message( target, ' No working rabbitMQ node could be found', 2) return False self.log_message(target, ' RabbitMQ test OK', 0) self.log_message(target, 'All tests OK', 0) return True except Exception as ex: self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2) return False
class Watcher(object): """ Watcher class """ LOG_CONTENTS = None def __init__(self, target, mode, level=0): # type: () -> None self._logger = Logger('extensions-generic') self.target = target self.mode = mode self.level = level # Minimal level to log def log_message(self, entry, level=None, show_level=0): # type: (str, int, int) -> None """ Logs an entry if above threshold level """ level = level or self.level # Picks self.level if not overriden per call if level >= show_level: # 0 = debug, 1 = info, 2 = error self._logger.debug('[{0}] {1}'.format(self.target, entry)) def _test_store(self, store_type, key=None, value=None): # type: (str, str, str) -> bool """ Test specified store type :param store_type: name of the store type :type: str :param key: key content to test :type key: str :param value: value to put :type value: str :return: boolean """ # Volatile self.log_message('Testing {0} store...'.format(store_type)) max_tries = 5 tries = 0 while tries < max_tries: if store_type == 'arakoon_voldrv': try: cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|voldrv')) configuration = Configuration.get('/ovs/arakoon/{0}/config'.format(cluster_name), raw=True) client = PyrakoonStore(cluster=cluster_name, configuration=configuration) client.nop() break except Exception as message: self.log_message(' Error during arakoon (voldrv) test: {0}'.format(message), 2) else: try: if store_type == 'volatile': VolatileFactory.store = None volatile = VolatileFactory.get_client() volatile.set(key, value) if volatile.get(key) == value: volatile.delete(key) break volatile.delete(key) elif store_type == 'persistent': persistent = PersistentFactory.get_client() persistent.nop() break except Exception as message: self.log_message(' Error during {0} store test: {1}'.format(store_type, message), 3) key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key time.sleep(1) tries += 1 if tries == max_tries: self.log_message(' {0} store not working correctly'.format(store_type), 2) return False self.log_message(' {0} store OK after {1} tries'.format(store_type, tries)) def services_running(self): # type: () -> bool """ Check if all services are running :return: Boolean """ try: key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) value = str(time.time()) if self.target in [WatcherTypes.CONFIG, WatcherTypes.FWK]: self.log_message('Testing configuration store...') try: Configuration.list('/') except Exception as ex: self.log_message(' Error during configuration store test: {0}'.format(ex), 2) return False with open(CACC_LOCATION) as config_file: contents = config_file.read() config = ArakoonClusterConfig(cluster_id=ARAKOON_NAME, load_config=False) config.read_config(contents=contents) client = ArakoonInstaller.build_client(config) contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY, consistency=NoGuarantee()) if Watcher.LOG_CONTENTS != contents: try: config.read_config(contents=contents) # Validate whether the contents are not corrupt except Exception as ex: self.log_message(' Configuration stored in configuration store seems to be corrupt: {0}'.format(ex), 2) return False temp_filename = '{0}~'.format(CACC_LOCATION) with open(temp_filename, 'w') as config_file: config_file.write(contents) config_file.flush() os.fsync(config_file) os.rename(temp_filename, CACC_LOCATION) Watcher.LOG_CONTENTS = contents self.log_message(' Configuration store OK', 0) if self.target == WatcherTypes.FWK: self._test_store('volatile', key, value) self._test_store('persistent') if self.target == WatcherTypes.VOLDRV: # Arakoon, voldrv cluster self._test_store('arakoon_voldrv') if self.target in [WatcherTypes.FWK, WatcherTypes.VOLDRV]: # RabbitMQ self.log_message('Test rabbitMQ...', 0) messagequeue = Configuration.get('/ovs/framework/messagequeue') rmq_servers = messagequeue['endpoints'] good_node = False for server in rmq_servers: try: connection_string = '{0}://{1}:{2}@{3}/%2F'.format(messagequeue['protocol'], messagequeue['user'], messagequeue['password'], server) connection = pika.BlockingConnection(pika.URLParameters(connection_string)) channel = connection.channel() channel.basic_publish('', 'ovs-watcher', str(time.time()), pika.BasicProperties(content_type='text/plain', delivery_mode=1)) connection.close() good_node = True except Exception as message: self.log_message(' Error during rabbitMQ test on node {0}: {1}'.format(server, message), 2) if good_node is False: self.log_message(' No working rabbitMQ node could be found', 2) return False self.log_message(' RabbitMQ test OK') self.log_message('All tests OK') return True except Exception as ex: self.log_message('Unexpected exception: {0}'.format(ex), 2) return False def start(self): # type: () -> None """ Start the ovs framework watcher """ if self.mode == 'wait': watcher.log_message('Waiting for master services', 1) while True: if watcher.services_running(): watcher.log_message('Master services available', 1) sys.exit(0) time.sleep(5) if self.mode == 'check': watcher.log_message('Checking master services', 1) while True: if not watcher.services_running(): watcher.log_message('One of the master services is unavailable', 1) sys.exit(1) time.sleep(5) watcher.log_message('Invalid parameter', 1) time.sleep(60) sys.exit(1)
class DistributedScheduler(Scheduler): """ Distributed scheduler that can run on multiple nodes at the same time. """ TIMEOUT = 60 * 30 def __init__(self, *args, **kwargs): """ Initializes the distributed scheduler """ self._mutex = volatile_mutex('celery_beat', 10) self._logger = Logger('celery') self._has_lock = False self._lock_name = 'ovs_celery_beat_lock' self._entry_name = 'ovs_celery_beat_entries' self._persistent = PersistentFactory.get_client() self._schedule_info = {} super(DistributedScheduler, self).__init__(*args, **kwargs) self._logger.debug('DS init') def setup_schedule(self): """ Setups the schedule """ self._logger.debug('DS setting up schedule') self._load_schedule() self.merge_inplace(self._discover_schedule()) self.install_default_entries(self.schedule) for schedule, source in self._schedule_info.iteritems(): self._logger.debug('* {0} ({1})'.format(schedule, source)) self._logger.debug('DS setting up schedule - done') def _discover_schedule(self): schedules = {} self._schedule_info = {} path = '/'.join([os.path.dirname(__file__), 'lib']) for filename in os.listdir(path): if os.path.isfile('/'.join([ path, filename ])) and filename.endswith('.py') and filename != '__init__.py': name = filename.replace('.py', '') mod = imp.load_source(name, '/'.join([path, filename])) for member in inspect.getmembers(mod, predicate=inspect.isclass): if member[1].__module__ == name: for submember in inspect.getmembers(member[1]): if hasattr(submember[1], 'schedule') and isinstance( submember[1].schedule, Schedule): schedule, source = submember[ 1].schedule.generate_schedule( submember[1].name) if schedule is not None: schedules[submember[1].name] = { 'task': submember[1].name, 'schedule': schedule, 'args': [] } self._schedule_info[submember[1].name] = source return schedules def _load_schedule(self): """ Loads the most recent schedule from the persistent store """ self.schedule = {} try: self._logger.debug('DS loading schedule entries') self._mutex.acquire(wait=10) try: self.schedule = cPickle.loads( str(self._persistent.get(self._entry_name))) except: # In case an exception occurs during loading the schedule, it is ignored and the default schedule # will be used/restored. pass finally: self._mutex.release() def sync(self): if self._has_lock is True: try: self._logger.debug('DS syncing schedule entries') self._mutex.acquire(wait=10) self._persistent.set(key=self._entry_name, value=cPickle.dumps(self.schedule)) except ArakoonSockNotReadable: self._logger.exception( 'Syncing the schedule failed this iteration') finally: self._mutex.release() else: self._logger.debug('DS skipping sync: lock is not ours') def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._logger.debug('DS executing tick') try: self._has_lock = False with self._mutex: # noinspection PyProtectedMember node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = System.get_my_machine_id() try: lock = self._persistent.get(self._lock_name) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True self._logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock self._logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock[ 'timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen self._logger.debug( 'DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) self._logger.debug('DS stealing lock from {0}'.format( lock['name'])) self._load_schedule() self._has_lock = True else: self._logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} self._logger.debug('DS refreshing lock') self._persistent.set(self._lock_name, lock) if self._has_lock is True: self._logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due( entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass self._logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval except Exception as ex: self._logger.debug('DS got error during tick: {0}'.format(ex)) return self.max_interval
def run(command, config=None, named_params=None, extra_params=None, client=None, debug=False, to_json=True): """ Executes a command on ALBA When --to-json is NOT passed: * An error occurs --> exitcode != 0 * It worked --> exitcode == 0 When --to-json is passed: * An errors occurs during verification of parameters passed -> exitcode != 0 * An error occurs while executing the command --> exitcode == 0 (error in json output) * It worked --> exitcode == 0 :param command: The command to execute, eg: 'list-namespaces' :type command: str :param config: The configuration location to be used, eg: 'arakoon://config/ovs/arakoon/ovsdb/config?ini=%2Fopt%2FOpenvStorage%2Fconfig%2Farakoon_cacc.ini' :type config: str :param named_params: Additional parameters to be given to the command, eg: {'long-id': ','.join(asd_ids)} :type named_params: dict :param extra_params: Additional parameters to be given to the command, eg: [name] :type extra_params: list :param client: A client on which to execute the command :type client: ovs_extensions.generic.sshclient.SSHClient :param debug: Log additional output :type debug: bool :param to_json: Request a JSON response from Alba :type to_json: bool :return: The output of the command :rtype: dict """ if named_params is None: named_params = {} if extra_params is None: extra_params = [] logger = Logger('extensions-plugins') if os.environ.get('RUNNING_UNITTESTS') == 'True': # For the unittest, all commands are passed to a mocked Alba from ovs.extensions.plugins.tests.alba_mockups import VirtualAlbaBackend named_params.update({'config': config}) named_params.update({'extra_params': extra_params}) return getattr(VirtualAlbaBackend, command.replace('-', '_'))(**named_params) debug_log = [] try: if to_json is True: extra_options = ["--to-json"] else: extra_options = [] cmd_list = ['/usr/bin/alba', command] + extra_options if config is not None: cmd_list.append('--config={0}'.format(config)) for key, value in named_params.iteritems(): cmd_list.append('--{0}={1}'.format(key, value)) cmd_list.extend(extra_params) cmd_string = ' '.join(cmd_list) debug_log.append('Command: {0}'.format(cmd_string)) start = time.time() try: if debug is True: logger.debug('Command: {0}'.format(cmd_string)) if client is None: try: if not hasattr(select, 'poll'): import subprocess subprocess._has_poll = False # Damn 'monkey patching' channel = Popen(cmd_list, stdout=PIPE, stderr=PIPE, universal_newlines=True) except OSError as ose: raise CalledProcessError(1, cmd_string, str(ose)) output, stderr = channel.communicate() output = re.sub(r'[^\x00-\x7F]+', '', output) stderr_debug = 'stderr: {0}'.format(stderr) stdout_debug = 'stdout: {0}'.format(output) if debug is True: logger.debug(stderr_debug) logger.debug(stdout_debug) debug_log.append(stderr_debug) debug_log.append(stdout_debug) exit_code = channel.returncode if exit_code != 0: # Raise same error as check_output raise CalledProcessError(exit_code, cmd_string, output) else: if debug is True: output, stderr = client.run(cmd_list, debug=True, return_stderr=True) debug_log.append('stderr: {0}'.format(stderr)) else: output = client.run(cmd_list).strip() debug_log.append('stdout: {0}'.format(output)) if to_json is True: output = json.loads(output) else: return output duration = time.time() - start if duration > 0.5: logger.warning('AlbaCLI call {0} took {1}s'.format( command, round(duration, 2))) except CalledProcessError as cpe: try: output = json.loads(cpe.output) except Exception: raise RuntimeError( 'Executing command {0} failed with output {1}'.format( cmd_string, cpe.output)) if output['success'] is True: return output['result'] raise AlbaError(output['error']['message'], output['error']['exception_code'], output['error']['exception_type']) except Exception as ex: logger.exception('Error: {0}'.format(ex)) # In case there's an exception, we always log for debug_line in debug_log: logger.debug(debug_line) raise