def __init__(self, object_type, query, key=None):
"""
Initializes a DataList class with a given key (used for optional caching) and a given query
:param object_type: The type of the objects that have to be queried
:param query: The query to execute
:param key: A key under which the result must be cached
"""
super(DataList, self).__init__()
if key is not None:
self._key = '{0}_{1}'.format(DataList.NAMESPACE, key)
else:
identifier = copy.deepcopy(query)
identifier['object'] = object_type.__name__
self._key = '{0}_{1}'.format(
DataList.NAMESPACE,
hashlib.sha256(json.dumps(identifier)).hexdigest())
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._query = query
self._can_cache = True
self._object_type = object_type
self._data = {}
self._objects = {}
self._guids = None
self._executed = False
self._shallow_sort = True
self.from_cache = None
def migrate():
"""
Executes all migrations. It keeps track of an internal "migration version" which is
a always increasing by one
"""
key = 'ovs_model_version'
persistent = PersistentFactory.get_client()
if persistent.exists(key):
data = persistent.get(key)
else:
data = {}
migrators = []
path = '/'.join([os.path.dirname(__file__), 'migration'])
for filename in os.listdir(path):
if os.path.isfile('/'.join([path, filename])) and filename.endswith('.py'):
name = filename.replace('.py', '')
mod = imp.load_source(name, '/'.join([path, filename]))
for member in inspect.getmembers(mod, predicate=inspect.isclass):
if member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]:
migrators.append((member[1].identifier, member[1].migrate, member[1].THIS_VERSION))
for identifier, method, end_version in migrators:
start_version = data.get(identifier, 0)
if end_version > start_version:
data[identifier] = method(start_version)
persistent.set(key, data)
def _set(key, value, raw):
key = EtcdConfiguration._coalesce_dashes(key=key)
data = value
if raw is False:
data = json.dumps(value)
# Unittests
if hasattr(unittest, 'running_tests') and getattr(unittest, 'running_tests') is True:
stripped_key = key.strip('/')
ut_data = EtcdConfiguration._unittest_data
for part in stripped_key.split('/')[:-1]:
if part not in ut_data:
ut_data[part] = {}
ut_data = ut_data[part]
ut_data[stripped_key.split('/')[-1]] = data
return
# Real implementation
client = EtcdConfiguration._get_client()
client.write(key, data)
try:
def _escape(*args, **kwargs):
_ = args, kwargs
raise RuntimeError()
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
signal.signal(signal.SIGALRM, _escape)
signal.alarm(0.5) # Wait only 0.5 seconds. This is a backup and should not slow down the system
client.set(key, value)
signal.alarm(0)
except:
pass
def get_pks(namespace, name):
"""
This method will load the primary keys for a given namespace and name
"""
persistent = PersistentFactory.get_client()
prefix = '{0}_{1}_'.format(namespace, name)
return set([key.replace(prefix, '') for key in persistent.prefix(prefix, max_elements=-1)])
def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid):
"""
This method will get a DataList for a relation.
On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
update all relations in the mean time.
For below parameter information, use following example: We called "my_vmachine.vdisks".
:param remote_class: The class of the remote part of the relation (e.g. VDisk)
:param remote_key: The key in the remote_class that points to us (e.g. vmachine)
:param own_class: The class of the base object of the relation (e.g. VMachine)
:param own_key: The key in this class pointing to the remote classes (e.g. vdisks)
:param own_guid: The guid of this object instance (e.g. the guid of my_vmachine)
"""
# Example:
# * remote_class = VDisk
# * remote_key = vmachine
# * own_class = VMachine
# * own_key = vdisks
# Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
# * own_guid = this vMachine object's guid
persistent = PersistentFactory.get_client()
own_name = own_class.__name__.lower()
datalist = DataList(remote_class, {}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key))
reverse_key = 'ovs_reverseindex_{0}_{1}|{2}|'.format(own_name, own_guid, own_key)
datalist._guids = [guid.replace(reverse_key, '') for guid in persistent.prefix(reverse_key)]
return datalist
def __init__(self, ensure_single_container, task):
"""
Initialize a EnsureSingle container
:param ensure_single_container: Ensure single arguments container
:type ensure_single_container: EnsureSingleContainer
:param task: Task instance
:type task: celery.AsyncResult
"""
self.ensure_single_container = ensure_single_container
# Storage
self.persistent_key = self.generate_key_for_task(
ensure_single_container.task_name)
self.persistent_client = PersistentFactory.get_client()
self.task_id, self.async_task = self.get_task_id_and_async(task)
# Logging
self.logger = Logger('lib')
# Runtime
self.now = None
self.thread_name = None
self.unittest_mode = None
self.message = None
self.runtime_hooks = {}
self.gather_run_time_info()
def _clean():
volatile = VolatileFactory.get_client()
persistent = PersistentFactory.get_client()
# noinspection PyProtectedMember
volatile._clean()
# noinspection PyProtectedMember
persistent._clean()
# noinspection PyProtectedMember
SSHClient._clean()
# noinspection PyProtectedMember
SystemdMock._clean()
# noinspection PyProtectedMember
MDSClient._clean()
# noinspection PyProtectedMember
Decorators._clean()
# noinspection PyProtectedMember
MockedSSHClient._clean()
# noinspection PyProtectedMember
StorageRouterClient._clean()
Logger._logs = {}
DataList._test_hooks = {}
Toolbox._function_pointers = {}
# Clean underlying persistent store
Configuration.get_client()._clean()
for file_name in glob.glob(
ArakoonClusterConfig.CONFIG_FILE.format('unittest*')):
os.remove(file_name)
for full_path in glob.glob(DalHelper.UNITTEST_DIR.format('*')):
shutil.rmtree(full_path)
return volatile, persistent
def _get(key):
if key == 'ovs.core.storage.persistent':
return 'arakoon'
c = PersistentFactory.get_client()
if c.exists(key):
return c.get(key)
return None
def get_relation_set(remote_class, remote_key, own_class, own_key,
own_guid):
"""
This method will get a DataList for a relation.
On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
update all relations in the mean time.
For below parameter information, use following example: We called "my_vmachine.vdisks".
:param remote_class: The class of the remote part of the relation (e.g. VDisk)
:param remote_key: The key in the remote_class that points to us (e.g. vmachine)
:param own_class: The class of the base object of the relation (e.g. VMachine)
:param own_key: The key in this class pointing to the remote classes (e.g. vdisks)
:param own_guid: The guid of this object instance (e.g. the guid of my_vmachine)
"""
# Example:
# * remote_class = VDisk
# * remote_key = vmachine
# * own_class = VMachine
# * own_key = vdisks
# Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
# * own_guid = this vMachine object's guid
persistent = PersistentFactory.get_client()
own_name = own_class.__name__.lower()
datalist = DataList(
remote_class, {}, '{0}_{1}_{2}'.format(own_name, own_guid,
remote_key))
reverse_key = 'ovs_reverseindex_{0}_{1}|{2}|'.format(
own_name, own_guid, own_key)
datalist._guids = [
guid.replace(reverse_key, '')
for guid in persistent.prefix(reverse_key)
]
return datalist
def __init__(self, object_type, query, key=None):
"""
Initializes a DataList class with a given key (used for optional caching) and a given query
:param object_type: The type of the objects that have to be queried
:param query: The query to execute
:param key: A key under which the result must be cached
"""
super(DataList, self).__init__()
if key is not None:
self._key = '{0}_{1}'.format(DataList.NAMESPACE, key)
else:
identifier = copy.deepcopy(query)
identifier['object'] = object_type.__name__
self._key = '{0}_{1}'.format(DataList.NAMESPACE, hashlib.sha256(json.dumps(identifier)).hexdigest())
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._query = query
self._can_cache = True
self._object_type = object_type
self._data = {}
self._objects = {}
self._guids = None
self._executed = False
self._shallow_sort = True
self.from_cache = None
def _clean_cache():
ovs_logger.info('Executing celery "clear_cache" startup script...')
from ovs.lib.helpers.decorators import ENSURE_SINGLE_KEY
active = inspect().active()
active_tasks = []
if active is not None:
for tasks in active.itervalues():
active_tasks += [task['id'] for task in tasks]
cache = PersistentFactory.get_client()
for key in cache.prefix(ENSURE_SINGLE_KEY):
try:
with volatile_mutex(name=key, wait=5):
entry = cache.get(key)
values = entry.get('values', [])
new_values = []
for v in values:
task_id = v.get('task_id')
if task_id is not None and task_id in active_tasks:
new_values.append(v)
if len(new_values) > 0:
entry['values'] = new_values
cache.set(key, entry)
ovs_logger.info('Updated key {0}'.format(key))
else:
cache.delete(key)
ovs_logger.info('Deleted key {0}'.format(key))
except KeyNotFoundException:
pass
ovs_logger.info('Executing celery "clear_cache" startup script... done')
def get_pks(namespace, name):
"""
This method will load the primary keys for a given namespace and name
"""
persistent = PersistentFactory.get_client()
prefix = '{0}_{1}_'.format(namespace, name)
for key in persistent.prefix(prefix):
yield key.replace(prefix, '')
def get_pks(namespace, name):
"""
This method will load the primary keys for a given namespace and name
"""
persistent = PersistentFactory.get_client()
prefix = '{0}_{1}_'.format(namespace, name)
for key in persistent.prefix(prefix):
yield key.replace(prefix, '')
def _clean_cache():
ovs_logger.info('Executing celery "clear_cache" startup script...')
persistent = PersistentFactory.get_client()
persistent.apply_callback_transaction(_get_registration_update_transaction,
max_retries=5)
ovs_logger.info('Executing celery "clear_cache" startup script... done')
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
def __init__(self):
self._works = False
self._getepoch = lambda: int(time.time())
self._begintime = self._getepoch()
self._waitlimit = 1800
self._client = PersistentFactory.get_client('arakoon')
self._key = 'ensureworks'
self._valueguid = str(uuid.uuid4())
self._value = '{0}{1}'
def tearDownClass(cls):
"""
Clean up the unittest
"""
fakesleep.monkey_restore()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def tearDownClass(cls):
"""
Tear down changes made during setUpClass
"""
Configuration._unittest_data = {}
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
def tearDownClass(cls):
"""
Tear down changes made during setUpClass
"""
Configuration._unittest_data = {}
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._logger = LogHandler.get('celery', name='celery beat')
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = volatile_mutex('celery_beat', 10)
self._schedule_info = {}
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
self._logger.debug('DS init')
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._logger = LogHandler.get('celery', name='celery beat')
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = volatile_mutex('celery_beat', 10)
self._schedule_info = {}
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
self._logger.debug('DS init')
def _set(key, value, raw):
client = EtcdConfiguration._get_client()
data = value
if raw is False:
data = json.dumps(value)
client.write(key, data)
try:
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
client.set(key, value)
except:
pass
def _test_store(self, store_type, key=None, value=None):
# type: (str, str, str) -> bool
"""
Test specified store type
:param store_type: name of the store type
:type: str
:param key: key content to test
:type key: str
:param value: value to put
:type value: str
:return: boolean
"""
# Volatile
self.log_message('Testing {0} store...'.format(store_type))
max_tries = 5
tries = 0
while tries < max_tries:
if store_type == 'arakoon_voldrv':
try:
cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|voldrv'))
configuration = Configuration.get('/ovs/arakoon/{0}/config'.format(cluster_name), raw=True)
client = PyrakoonStore(cluster=cluster_name, configuration=configuration)
client.nop()
break
except Exception as message:
self.log_message(' Error during arakoon (voldrv) test: {0}'.format(message), 2)
else:
try:
if store_type == 'volatile':
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
elif store_type == 'persistent':
persistent = PersistentFactory.get_client()
persistent.nop()
break
except Exception as message:
self.log_message(' Error during {0} store test: {1}'.format(store_type, message), 3)
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(' {0} store not working correctly'.format(store_type), 2)
return False
self.log_message(' {0} store OK after {1} tries'.format(store_type, tries))
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
StorageRouterClient.clean()
fakesleep.monkey_patch()
Configuration.set('/ovs/framework/storagedriver|mds_tlogs', 100)
Configuration.set('/ovs/framework/storagedriver|mds_maxload', 75)
Configuration.set('/ovs/framework/storagedriver|mds_safety', 2)
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
StorageRouterClient.clean()
fakesleep.monkey_patch()
Configuration.set('/ovs/framework/arakoon_clusters|voldrv', 'voldrv')
Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]})
Configuration.set('/ovs/framework/rdma', False)
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
StorageRouterClient.clean()
fakesleep.monkey_patch()
Configuration.set('/ovs/framework/storagedriver|mds_tlogs', 100)
Configuration.set('/ovs/framework/storagedriver|mds_maxload', 75)
Configuration.set('/ovs/framework/storagedriver|mds_safety', 2)
def __init__(self):
"""
Init
"""
self.persistent = PersistentFactory.get_client()
trap_host_port = "{}_config_trap_target".format(STORAGE_PREFIX)
self.sender = None
try:
target_host, port = self.persistent.get(trap_host_port)
except KeyNotFoundException:
print('OVS SNMP Target not configured, cannot send TRAP')
else:
self.sender = SNMPTrapSender(target_host, port)
# security from model
self.sender.security_public()
def try_get(key, fallback):
"""
Returns a value linked to a certain key from the volatile store.
If not found in the volatile store, it will try fetch it from the persistent
store. If not found, it returns the fallback
"""
volatile = VolatileFactory.get_client()
data = volatile.get(key)
if data is None:
try:
persistent = PersistentFactory.get_client()
data = persistent.get(key)
except:
data = fallback
volatile.set(key, data)
return data
def try_get(key, fallback):
"""
Returns a value linked to a certain key from the volatile store.
If not found in the volatile store, it will try fetch it from the persistent
store. If not found, it returns the fallback
"""
volatile = VolatileFactory.get_client()
data = volatile.get(key)
if data is None:
try:
persistent = PersistentFactory.get_client()
data = persistent.get(key)
except:
data = fallback
volatile.set(key, data)
return data
def __init__(self):
"""
Init
"""
self.persistent = PersistentFactory.get_client()
trap_host_port = "{}_config_trap_target".format(STORAGE_PREFIX)
self.sender = None
try:
target_host, port = self.persistent.get(trap_host_port)
except KeyNotFoundException:
print('OVS SNMP Target not configured, cannot send TRAP')
else:
self.sender = SNMPTrapSender(target_host, port)
# security from model
self.sender.security_public()
def __init__(self, port):
"""
Init
"""
signal.signal(signal.SIGTERM, self.SIGTERM)
from ovs.extensions.generic.system import System
my_storagerouter = System.get_my_storagerouter()
self.host = my_storagerouter.ip
self.port = port
self.persistent = PersistentFactory.get_client()
self.users = self.get_users()
# Load from model
self.assigned_oids = {}
self.instance_oid = 0
# Book-keeping
self.model_oids = set()
def _set(key, value, raw):
client = EtcdConfiguration._get_client()
data = value
if raw is False:
data = json.dumps(value)
client.write(key, data)
try:
def _escape(*args, **kwargs):
_ = args, kwargs
raise RuntimeError()
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
signal.signal(signal.SIGALRM, _escape)
signal.alarm(0.5) # Wait only 0.5 seconds. This is a backup and should not slow down the system
client.set(key, value)
signal.alarm(0)
except:
pass
def verify_arakoon(arakoon_cluster_name):
"""
Tries a nop on a arakoon cluster
:param arakoon_cluster_name: name of the arakoon cluster
:return: None
"""
pyrakoon_client = PersistentFactory.get_client('pyrakoon')
try:
# determine if there is a healthy cluster
pyrakoon_client.nop()
result_handler.success('Arakoon {0} responded successfully.'.format(arakoon_cluster_name))
except ArakoonNotFound as ex:
result_handler.failure('Arakoon {0} seems to be down. Got {1}'.format(arakoon_cluster_name, str(ex)))
except (ArakoonNoMaster, ArakoonNoMasterResult) as ex:
result_handler.failure('Arakoon {0} cannot find a master. Got {1}'.format(arakoon_cluster_name, str(ex)))
except TimeoutError:
result_handler.warning('Arakoon {0} did not respond within {1}s'.format(arakoon_cluster_name, ArakoonHealthCheck.INTEGRITY_TIMEOUT))
except Exception as ex:
result_handler.exception('Arakoon {0} could not process a nop. Got {1}'.format(arakoon_cluster_name, str(ex)))
def __init__(self, object_type, query=None, key=None, guids=None):
"""
Initializes a DataList class with a given key (used for optional caching) and a given query
:param object_type: The type of the objects that have to be queried
:param query: The query to execute. Example: {'type': DataList.where_operator.AND, 'items': [('storagedriver_id', DataList.operator.EQUALS, storagedriver_id)]}
When query is None, it will default to a query which will not do any filtering
:type query: dict or NoneType
:param key: A key under which the result must be cached
:type key: str
:param guids: List of guids to use as a base
These guids should be guids of objects related to the object_type param. If no object related to the guid could be found, these guids will not be included in the result
When guids is None, it will default to querying all items
:type guids: list[basestring] or NoneType
"""
# Validation
self.validate_guids(guids)
self.validate_query(query)
# Defaults
if query is None:
query = {'type': DataList.where_operator.AND, 'items': []}
super(DataList, self).__init__()
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._query = query
self._can_cache = True
self._object_type = object_type
self._data = {}
self._objects = {}
self._guids = None
self._executed = False
self._shallow_sort = True
self._provided_guids = guids
self._provided_keys = None # Conversion of guids to keys, cached for faster lookup
self._key = None
self._provided_key = False # Keep track whether a key was explicitly set
self.from_cache = None
self.from_index = 'none'
self.set_key(key)
def _get_pks(namespace, name):
"""
Loads the primary key set information and pages, merges them to a single set
and returns it
"""
internal_key = 'ovs_primarykeys_{0}_{{0}}'.format(name)
volatile = VolatileFactory.get_client()
persistent = PersistentFactory.get_client()
pointer = internal_key.format(0)
keys = set()
while pointer is not None:
subset = volatile.get(pointer)
if subset is None:
prefix = '{0}_{1}_'.format(namespace, name)
keys = set([key.replace(prefix, '') for key in persistent.prefix(prefix, max_elements=-1)])
DataList._save_pks(name, keys)
return keys
keys = keys.union(subset[0])
pointer = subset[1]
return keys
def __init__(self,
app,
expires=None,
backend=None,
options=None,
url=None,
**kwargs):
super(ArakoonResultBackend, self).__init__(app, **kwargs)
if options is None:
options = {}
self.url = url
self.options = dict(self.app.conf.CELERY_CACHE_BACKEND_OPTIONS,
**options)
self.backend = url or backend or self.app.conf.CELERY_RESULT_BACKEND # Will be 'arakoon'
self.expires = self.prepare_expires(expires, type=int)
self._encode_prefixes() # rencode the keyprefixes
self._client = PersistentFactory.get_client()
def migrate():
"""
Executes all migrations. It keeps track of an internal "migration version" which is
a always increasing by one
"""
def execute(function, start, end):
"""
Executes a single migration, syncing versions
"""
version = function(start)
if version > end:
end = version
return end
key = 'ovs_model_version'
persistent = PersistentFactory.get_client()
if persistent.exists(key):
data = persistent.get(key)
else:
data = {}
migrators = []
path = os.path.join(os.path.dirname(__file__), 'migration')
for filename in os.listdir(path):
if os.path.isfile(os.path.join(
path, filename)) and filename.endswith('.py'):
name = filename.replace('.py', '')
module = imp.load_source(name, os.path.join(path, filename))
for member in inspect.getmembers(module):
if inspect.isclass(member[1]) \
and member[1].__module__ == name \
and 'object' in [base.__name__ for base in member[1].__bases__]:
migrators.append(
(member[1].identifier, member[1].migrate))
for identifier, method in migrators:
base_version = data[identifier] if identifier in data else 0
new_version = execute(method, base_version, 0)
data[identifier] = new_version
persistent.set(key, data)
def migrate():
"""
Executes all migrations. It keeps track of an internal "migration version" which is
a always increasing by one
"""
def execute(function, start, end):
"""
Executes a single migration, syncing versions
"""
version = function(start)
if version > end:
end = version
return end
key = 'ovs_model_version'
persistent = PersistentFactory.get_client()
if persistent.exists(key):
data = persistent.get(key)
else:
data = {}
migrators = []
path = '/'.join([os.path.dirname(__file__), 'migration'])
for filename in os.listdir(path):
if os.path.isfile('/'.join([path, filename])) and filename.endswith('.py'):
name = filename.replace('.py', '')
module = imp.load_source(name, '/'.join([path, filename]))
for member in inspect.getmembers(module):
if inspect.isclass(member[1]) \
and member[1].__module__ == name \
and 'object' in [base.__name__ for base in member[1].__bases__]:
migrators.append((member[1].identifier, member[1].migrate))
for identifier, method in migrators:
base_version = data[identifier] if identifier in data else 0
new_version = execute(method, base_version, 0)
data[identifier] = new_version
persistent.set(key, data)
def __init__(self, query, key=None, load=True):
"""
Initializes a DataList class with a given key (used for optional caching) and a given query
"""
# Initialize super class
super(DataList, self).__init__()
if key is not None:
self._key = key
else:
identifier = copy.deepcopy(query)
identifier['object'] = identifier['object'].__name__
self._key = hashlib.sha256(json.dumps(identifier)).hexdigest()
self._key = '{0}_{1}'.format(DataList.namespace, self._key)
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._query = query
self.data = None
self.from_cache = False
self._can_cache = True
if load is True:
self._load()
def _set(key, value, raw):
key = EtcdConfiguration._coalesce_dashes(key=key)
data = value
if raw is False:
data = json.dumps(value)
# Unittests
if hasattr(unittest, 'running_tests') and getattr(
unittest, 'running_tests') is True:
stripped_key = key.strip('/')
ut_data = EtcdConfiguration._unittest_data
for part in stripped_key.split('/')[:-1]:
if part not in ut_data:
ut_data[part] = {}
ut_data = ut_data[part]
ut_data[stripped_key.split('/')[-1]] = data
return
# Real implementation
client = EtcdConfiguration._get_client()
client.write(key, data)
try:
def _escape(*args, **kwargs):
_ = args, kwargs
raise RuntimeError()
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
signal.signal(signal.SIGALRM, _escape)
signal.alarm(
0.5
) # Wait only 0.5 seconds. This is a backup and should not slow down the system
client.set(key, value)
signal.alarm(0)
except:
pass
def _get_pks(namespace, name):
"""
Loads the primary key set information and pages, merges them to a single set
and returns it
"""
internal_key = 'ovs_primarykeys_{0}_{{0}}'.format(name)
volatile = VolatileFactory.get_client()
persistent = PersistentFactory.get_client()
pointer = internal_key.format(0)
keys = set()
while pointer is not None:
subset = volatile.get(pointer)
if subset is None:
prefix = '{0}_{1}_'.format(namespace, name)
keys = set([
key.replace(prefix, '')
for key in persistent.prefix(prefix, max_elements=-1)
])
DataList._save_pks(name, keys)
return keys
keys = keys.union(subset[0])
pointer = subset[1]
return keys
def __init__(self, vdisk_guid):
# type: (str) -> None
"""
Initializes a new MDSCatchUp
An instance populates some caches. These cached are cleared once the instance is garbage collected.
When running MDSCatchup in bulk: add them to a list to speed up the process
:param vdisk_guid: Guid of the vDisk to catch up for
:type vdisk_guid: str
"""
self.id = str(uuid.uuid4())
self.vdisk = VDisk(vdisk_guid)
self.mds_key = self._CATCH_UP_VDISK_KEY.format(self.vdisk.guid)
self.tlog_threshold = Configuration.get(
'ovs/volumedriver/mds|tlogs_behind', default=100)
self.volumedriver_service_name = 'ovs-volumedriver_{0}'.format(
self.vdisk.vpool.name)
self.mds_client_timeout = Configuration.get(
'ovs/vpools/{0}/mds_config|mds_client_connection_timeout'.format(
self.vdisk.vpool_guid),
default=120)
self.mds_clients = {}
self.dry_run = False
self.catch_up_threads = []
self.errors = []
self._service_manager = ServiceFactory.get_manager()
self._persistent = PersistentFactory.get_client()
self._log = 'MDS catchup {0} - vDisk {1} (volume id: {2})'.format(
self.id, self.vdisk.guid, self.vdisk.volume_id)
self._clients = self.build_clients()
self._volumedriver_contexts = self.get_volumedriver_contexts()
self._worker_contexts = self.get_worker_contexts()
self._worker_context = self._worker_contexts[
System.get_my_storagerouter()]
self._relevant_contexts = self._get_all_relevant_contexts(
) # All possible contexts (by mixing volumedriver ones with workers)
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.factory = None
PersistentFactory.get_client().clean()
VolatileFactory.get_client().clean()
admin_group = Group()
admin_group.name = 'administrators'
admin_group.description = 'Administrators'
admin_group.save()
viewers_group = Group()
viewers_group.name = 'viewers'
viewers_group.description = 'Viewers'
viewers_group.save()
# Create users
admin = User()
admin.username = '******'
admin.password = hashlib.sha256('admin').hexdigest()
admin.is_active = True
admin.group = admin_group
admin.save()
admin_npg = User()
admin_npg.username = '******'
admin_npg.password = hashlib.sha256('admin_npg').hexdigest()
admin_npg.is_active = True
admin_npg.group = admin_group
admin_npg.save()
admin_na = User()
admin_na.username = '******'
admin_na.password = hashlib.sha256('admin_na').hexdigest()
admin_na.is_active = False
admin_na.group = admin_group
admin_na.save()
user = User()
user.username = '******'
user.password = hashlib.sha256('user').hexdigest()
user.is_active = True
user.group = viewers_group
user.save()
# Create internal OAuth 2 clients
admin_client = Client()
admin_client.ovs_type = 'INTERNAL'
admin_client.grant_type = 'PASSWORD'
admin_client.user = admin
admin_client.save()
admin_na_client = Client()
admin_na_client.ovs_type = 'INTERNAL'
admin_na_client.grant_type = 'PASSWORD'
admin_na_client.user = admin_na
admin_na_client.save()
user_client = Client()
user_client.ovs_type = 'INTERNAL'
user_client.grant_type = 'PASSWORD'
user_client.user = user
user_client.save()
# Create roles
read_role = Role()
read_role.code = 'read'
read_role.name = 'Read'
read_role.description = 'Can read objects'
read_role.save()
write_role = Role()
write_role.code = 'write'
write_role.name = 'Write'
write_role.description = 'Can write objects'
write_role.save()
manage_role = Role()
manage_role.code = 'manage'
manage_role.name = 'Manage'
manage_role.description = 'Can manage the system'
manage_role.save()
# Attach groups to roles
mapping = [(admin_group, [read_role, write_role, manage_role]),
(viewers_group, [read_role])]
for setting in mapping:
for role in setting[1]:
rolegroup = RoleGroup()
rolegroup.group = setting[0]
rolegroup.role = role
rolegroup.save()
for user in setting[0].users:
for role in setting[1]:
for client in user.clients:
roleclient = RoleClient()
roleclient.client = client
roleclient.role = role
roleclient.save()
storagerouter = StorageRouter()
storagerouter.machine_id = 'storagerouter'
storagerouter.ip = '127.0.0.1'
storagerouter.machine_id = '1'
storagerouter.rdma_capable = False
storagerouter.name = 'storagerouter'
storagerouter.save()
from django.test import RequestFactory
cls.factory = RequestFactory()
fakesleep.monkey_patch()
Configuration.set('/ovs/framework/plugins/installed', {
'generic': [],
'backends': []
})
Configuration.set('/ovs/framework/cluster_id', 'cluster_id')
System._machine_id = {'none': '1'}
@worker_process_init.connect
def worker_process_init_handler(args=None, kwargs=None, **kwds):
"""
Hook for process init
"""
_ = args, kwargs, kwds
VolatileFactory.store = None
PersistentFactory.store = None
ArakoonConfiguration.client = None
@after_setup_task_logger.connect
@after_setup_logger.connect
def load_ovs_logger(**kwargs):
"""Load a logger."""
if 'logger' in kwargs:
kwargs['logger'] = LogHandler.get('celery', name='celery')
if __name__ == '__main__':
import sys
if len(sys.argv) == 2 and sys.argv[1] == 'clear_cache':
from ovs.lib.helpers.decorators import ENSURE_SINGLE_KEY
cache = PersistentFactory.get_client()
for key in cache.prefix(ENSURE_SINGLE_KEY):
try:
cache.delete(key)
except KeyNotFoundException:
pass
def test_ensure_safety(self):
"""
Validates whether the ensure_safety call works as expected
"""
def _generate_mds_service_load_repr(_mds_service):
"""
Generates a load representing thing for a given mds_service
"""
masters, slaves = 0, 0
for _junction in _mds_service.vdisks:
if _junction.is_master:
masters += 1
else:
slaves += 1
capacity = _mds_service.capacity
if capacity == -1:
capacity = 'infinite'
_load, _ = MDSServiceController.get_mds_load(_mds_service)
if _load == float('inf'):
_load = 'infinite'
else:
_load = round(_load, 2)
return [_mds_service.service.storagerouter.ip, _mds_service.service.ports[0], masters, slaves, capacity, _load]
def _check_reality(_configs, _loads, _vdisks, _mds_services, test=True, display=False):
"""
Validates 'reality' with an expected config/load
"""
reality_configs = []
for _vdisk_id in _vdisks:
reality_configs.append(_vdisks[_vdisk_id].info['metadata_backend_config'])
if display is True:
for c in reality_configs:
print c
if test is True:
self.assertListEqual(reality_configs, _configs)
reality_loads = []
for mds_id in _mds_services:
reality_loads.append(_generate_mds_service_load_repr(_mds_services[mds_id]))
if display is True:
for l in reality_loads:
print l
if test is True:
self.assertListEqual(reality_loads, _loads)
PersistentFactory.get_client().set('ovs.storagedriver.mds.safety', 3)
PersistentFactory.get_client().set('ovs.storagedriver.mds.maxload', 75)
PersistentFactory.get_client().set('ovs.storagedriver.mds.tlogs', 100)
vpools, storagerouters, storagedrivers, _, mds_services, service_type = self._build_service_structure(
{'vpools': [1],
'storagerouters': [1, 2, 3, 4],
'storagedrivers': [(1, 1, 1), (2, 1, 2), (3, 1, 3), (4, 1, 4)], # (<id>, <vpool_id>, <sr_id>)
'mds_services': [(1, 1), (2, 2), (3, 3), (4, 4)]} # (<id>, <sd_id>)
)
vdisks = {}
start_id = 1
for mds_service in mds_services.itervalues():
vdisks.update(self._create_vdisks_for_mds_service(2, start_id, mds_service=mds_service))
start_id += 2
# Validate the start configuration which is simple, each disk has only its default local master
configs = [[{'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.2', 'port': 2}],
[{'ip': '10.0.0.2', 'port': 2}],
[{'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.4', 'port': 4}]]
loads = [['10.0.0.1', 1, 2, 0, 10, 20.0],
['10.0.0.2', 2, 2, 0, 10, 20.0],
['10.0.0.3', 3, 2, 0, 10, 20.0],
['10.0.0.4', 4, 2, 0, 10, 20.0]]
_check_reality(configs, loads, vdisks, mds_services)
# Validate first run. Each disk should now have sufficient nodes, since there are plenty of MDS services available
configs = [[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.2', 'port': 2}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.2', 'port': 2}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.2', 'port': 2}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}]]
loads = [['10.0.0.1', 1, 2, 5, 10, 70.0],
['10.0.0.2', 2, 2, 4, 10, 60.0],
['10.0.0.3', 3, 2, 4, 10, 60.0],
['10.0.0.4', 4, 2, 3, 10, 50.0]]
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# Validate whether this extra (unnecessary) run doesn't change anything, preventing reconfiguring over and
# over again
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# Validating whether an overloaded node is correctly rebalanced
mds_services[2].capacity = 2
mds_services[2].save()
configs = [[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}]]
loads = [['10.0.0.1', 1, 2, 5, 10, 70.0],
['10.0.0.2', 2, 2, 0, 2, 100.0],
['10.0.0.3', 3, 2, 5, 10, 70.0],
['10.0.0.4', 4, 2, 5, 10, 70.0]]
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# Validate whether the overloaded services are still handled. In this case, causing a re-order of the slaves as
# ordered in the model
configs = [[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}]]
loads = [['10.0.0.1', 1, 2, 5, 10, 70.0],
['10.0.0.2', 2, 2, 0, 2, 100.0],
['10.0.0.3', 3, 2, 5, 10, 70.0],
['10.0.0.4', 4, 2, 5, 10, 70.0]]
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# Again, validating whether a subsequent run doesn't give unexpected changes
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# A MDS service will be added (next to the overloaded service), this should cause the expected to be rebalanced
s_id = '{0}-5'.format(storagerouters[2].name)
service = Service()
service.name = s_id
service.storagerouter = storagerouters[2]
service.ports = [5]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = vpools[1]
mds_service.save()
mds_services[5] = mds_service
configs = [[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.2', 'port': 5}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.2', 'port': 5}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.2', 'port': 5}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}]]
loads = [['10.0.0.1', 1, 2, 5, 10, 70.0],
['10.0.0.2', 2, 2, 0, 2, 100.0],
['10.0.0.3', 3, 2, 5, 10, 70.0],
['10.0.0.4', 4, 2, 5, 10, 70.0],
['10.0.0.2', 5, 0, 3, 10, 30.0]]
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# If the tlogs are not caught up, nothing should be changed
for vdisk_id in [3, 4]:
StorageDriverClient.catch_up[vdisks[vdisk_id].volume_id] = 1000
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# The next run, after tlogs are caught up, a master switch should be executed
for vdisk_id in [3, 4]:
StorageDriverClient.catch_up[vdisks[vdisk_id].volume_id] = 50
configs = [[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 5}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.2', 'port': 5}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}]]
loads = [['10.0.0.1', 1, 2, 5, 10, 70.0],
['10.0.0.2', 2, 1, 0, 2, 50.0],
['10.0.0.3', 3, 2, 5, 10, 70.0],
['10.0.0.4', 4, 2, 5, 10, 70.0],
['10.0.0.2', 5, 1, 1, 10, 20.0]]
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# Validate whether a volume migration makes the master follow
StorageDriverClient.vrouter_id[vdisks[1].volume_id] = storagedrivers[3].storagedriver_id
configs = [[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.2', 'port': 5}],
[{'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 5}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.3', 'port': 3}],
[{'ip': '10.0.0.2', 'port': 2}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}, {'ip': '10.0.0.4', 'port': 4}],
[{'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.1', 'port': 1}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.2', 'port': 5}],
[{'ip': '10.0.0.4', 'port': 4}, {'ip': '10.0.0.3', 'port': 3}, {'ip': '10.0.0.1', 'port': 1}]]
loads = [['10.0.0.1', 1, 1, 6, 10, 70.0],
['10.0.0.2', 2, 1, 0, 2, 50.0],
['10.0.0.3', 3, 3, 4, 10, 70.0],
['10.0.0.4', 4, 2, 4, 10, 60.0],
['10.0.0.2', 5, 1, 2, 10, 30.0]]
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
# Validates if a second run doesn't change anything
for vdisk_id in sorted(vdisks.keys()):
MDSServiceController.ensure_safety(vdisks[vdisk_id])
_check_reality(configs, loads, vdisks, mds_services)
def test_storagedriver_config_set(self):
"""
Validates whether storagedriver configuration is generated as expected
"""
PersistentFactory.get_client().set('ovs.storagedriver.mds.safety', 3)
vpools, storagerouters, storagedrivers, services, mds_services, _ = self._build_service_structure(
{'vpools': [1, 2],
'storagerouters': [1, 2, 3, 4, 5, 6],
'storagedrivers': [(1, 1, 1), (2, 1, 2), (3, 1, 3), (4, 1, 4), (5, 2, 4), (6, 2, 5), (7, 2, 6)], # (<id>, <vpool_id>, <sr_id>)
'mds_services': [(1, 1), (2, 1), (3, 2), (4, 3), (5, 4), (6, 5), (7, 6), (8, 7), (9, 7)]} # (<id>, <sd_id>)
)
vdisks = {}
start_id = 1
for mds_service in mds_services.itervalues():
vdisks.update(self._create_vdisks_for_mds_service(10, start_id, mds_service=mds_service))
start_id += 10
mds_services[1].capacity = 11 # on 1, vpool 1
mds_services[1].save()
mds_services[2].capacity = 20 # on 1, vpool 1
mds_services[2].save()
mds_services[3].capacity = 12 # on 2, vpool 1
mds_services[3].save()
mds_services[4].capacity = 14 # on 3, vpool 1
mds_services[4].save()
mds_services[5].capacity = 16 # on 4, vpool 1
mds_services[5].save()
mds_services[6].capacity = 11 # on 4, vpool 2
mds_services[6].save()
mds_services[7].capacity = 13 # on 5, vpool 2
mds_services[7].save()
mds_services[8].capacity = 19 # on 6, vpool 2
mds_services[8].save()
mds_services[9].capacity = 15 # on 6, vpool 2
mds_services[9].save()
config = MDSServiceController.get_mds_storagedriver_config_set(vpools[1])
expected = {storagerouters[1].guid: [{'host': '10.0.0.1', 'port': 2},
{'host': '10.0.0.4', 'port': 5},
{'host': '10.0.0.3', 'port': 4}],
storagerouters[2].guid: [{'host': '10.0.0.2', 'port': 3},
{'host': '10.0.0.1', 'port': 2},
{'host': '10.0.0.4', 'port': 5}],
storagerouters[3].guid: [{'host': '10.0.0.3', 'port': 4},
{'host': '10.0.0.1', 'port': 2},
{'host': '10.0.0.4', 'port': 5}],
storagerouters[4].guid: [{'host': '10.0.0.4', 'port': 5},
{'host': '10.0.0.1', 'port': 2},
{'host': '10.0.0.3', 'port': 4}]}
self.assertDictEqual(config, expected, 'Test 1. Got:\n{0}'.format(json.dumps(config, indent=2)))
mds_services[2].capacity = 10 # on 1, vpool 1
mds_services[2].save()
config = MDSServiceController.get_mds_storagedriver_config_set(vpools[1])
expected = {storagerouters[1].guid: [{'host': '10.0.0.1', 'port': 1},
{'host': '10.0.0.4', 'port': 5},
{'host': '10.0.0.3', 'port': 4}],
storagerouters[2].guid: [{'host': '10.0.0.2', 'port': 3},
{'host': '10.0.0.4', 'port': 5},
{'host': '10.0.0.3', 'port': 4}],
storagerouters[3].guid: [{'host': '10.0.0.3', 'port': 4},
{'host': '10.0.0.4', 'port': 5},
{'host': '10.0.0.2', 'port': 3}],
storagerouters[4].guid: [{'host': '10.0.0.4', 'port': 5},
{'host': '10.0.0.3', 'port': 4},
{'host': '10.0.0.2', 'port': 3}]}
self.assertDictEqual(config, expected, 'Test 2. Got:\n{0}'.format(json.dumps(config, indent=2)))
def _get(key):
c = PersistentFactory.get_client()
if c.exists(key):
return c.get(key)
return None
def __init__(self):
"""
Init method
"""
self.persistent = PersistentFactory.get_client()
def migrate(previous_version):
"""
Migrates from a given version to the current version. It uses 'previous_version' to be smart
wherever possible, but the code should be able to migrate any version towards the expected version.
When this is not possible, the code can set a minimum version and raise when it is not met.
:param previous_version: The previous version from which to start the migration
:type previous_version: float
"""
working_version = previous_version
if working_version == 0:
from ovs.dal.hybrids.servicetype import ServiceType
# Initial version:
# * Add any basic configuration or model entries
# Add backends
for backend_type_info in [('ALBA', 'alba')]:
code = backend_type_info[1]
backend_type = BackendTypeList.get_backend_type_by_code(code)
if backend_type is None:
backend_type = BackendType()
backend_type.name = backend_type_info[0]
backend_type.code = code
backend_type.save()
# Add service types
for service_type_info in [
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR,
ServiceType.SERVICE_TYPES.ALBA_S3_TRANSACTION
]:
service_type = ServiceType()
service_type.name = service_type_info
service_type.save()
# From here on, all actual migration should happen to get to the expected state for THIS RELEASE
elif working_version < DALMigrator.THIS_VERSION:
import hashlib
from ovs.dal.exceptions import ObjectNotFoundException
from ovs.dal.helpers import HybridRunner, Descriptor
from ovs.dal.hybrids.albaabmcluster import ABMCluster
from ovs.dal.hybrids.albaosd import AlbaOSD
from ovs.dal.hybrids.albansmcluster import NSMCluster
from ovs.dal.hybrids.j_abmservice import ABMService
from ovs.dal.hybrids.j_nsmservice import NSMService
from ovs.dal.hybrids.service import Service
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.lists.albabackendlist import AlbaBackendList
from ovs.dal.lists.albanodelist import AlbaNodeList
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.extensions.db.arakooninstaller import ArakoonClusterConfig, ArakoonInstaller
from ovs.extensions.generic.configuration import Configuration, NotFoundException
from ovs_extensions.generic.toolbox import ExtensionsToolbox
from ovs.extensions.plugins.albacli import AlbaCLI
from ovs.extensions.storage.persistentfactory import PersistentFactory
# Migrate unique constraints & indexes
client = PersistentFactory.get_client()
hybrid_structure = HybridRunner.get_hybrids()
for class_descriptor in hybrid_structure.values():
cls = Descriptor().load(class_descriptor).get_object()
classname = cls.__name__.lower()
unique_key = 'ovs_unique_{0}_{{0}}_'.format(classname)
index_prefix = 'ovs_index_{0}|{{0}}|'.format(classname)
index_key = 'ovs_index_{0}|{{0}}|{{1}}'.format(classname)
uniques = []
indexes = []
# noinspection PyProtectedMember
for prop in cls._properties:
if prop.unique is True and len([
k for k in client.prefix(
unique_key.format(prop.name))
]) == 0:
uniques.append(prop.name)
if prop.indexed is True and len([
k for k in client.prefix(
index_prefix.format(prop.name))
]) == 0:
indexes.append(prop.name)
if len(uniques) > 0 or len(indexes) > 0:
prefix = 'ovs_data_{0}_'.format(classname)
for key, data in client.prefix_entries(prefix):
for property_name in uniques:
ukey = '{0}{1}'.format(
unique_key.format(property_name),
hashlib.sha1(str(
data[property_name])).hexdigest())
client.set(ukey, key)
for property_name in indexes:
if property_name not in data:
continue # This is the case when there's a new indexed property added.
ikey = index_key.format(
property_name,
hashlib.sha1(str(
data[property_name])).hexdigest())
index = list(
client.get_multi([ikey], must_exist=False))[0]
transaction = client.begin_transaction()
if index is None:
client.assert_value(ikey,
None,
transaction=transaction)
client.set(ikey, [key],
transaction=transaction)
elif key not in index:
client.assert_value(ikey,
index[:],
transaction=transaction)
client.set(ikey,
index + [key],
transaction=transaction)
client.apply_transaction(transaction)
#############################################
# Introduction of ABMCluster and NSMCluster #
#############################################
# Verify presence of unchanged ALBA Backends
alba_backends = AlbaBackendList.get_albabackends()
changes_required = False
for alba_backend in alba_backends:
if alba_backend.abm_cluster is None or len(
alba_backend.nsm_clusters) == 0:
changes_required = True
break
if changes_required:
# Retrieve ABM and NSM clusters
abm_cluster_info = []
nsm_cluster_info = []
for cluster_name in Configuration.list('/ovs/arakoon'):
try:
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=cluster_name)
if metadata[
'cluster_type'] == ServiceType.ARAKOON_CLUSTER_TYPES.ABM:
abm_cluster_info.append(metadata)
elif metadata[
'cluster_type'] == ServiceType.ARAKOON_CLUSTER_TYPES.NSM:
nsm_cluster_info.append(metadata)
except NotFoundException:
continue
# Retrieve NSM Arakoon cluster information
cluster_arakoon_map = {}
for cluster_info in abm_cluster_info + nsm_cluster_info:
cluster_name = cluster_info['cluster_name']
arakoon_config = ArakoonClusterConfig(
cluster_id=cluster_name)
cluster_arakoon_map[
cluster_name] = arakoon_config.export_dict()
storagerouter_map = dict(
(storagerouter.machine_id, storagerouter) for storagerouter
in StorageRouterList.get_storagerouters())
alba_backend_id_map = dict((alba_backend.alba_id, alba_backend)
for alba_backend in alba_backends)
for cluster_info in abm_cluster_info:
internal = cluster_info['internal']
cluster_name = cluster_info['cluster_name']
config_location = Configuration.get_configuration_path(
key=ArakoonClusterConfig.CONFIG_KEY.format(
cluster_name))
try:
alba_id = AlbaCLI.run(command='get-alba-id',
config=config_location,
named_params={'attempts':
3})['id']
nsm_hosts = AlbaCLI.run(command='list-nsm-hosts',
config=config_location,
named_params={'attempts': 3})
except RuntimeError:
continue
alba_backend = alba_backend_id_map.get(alba_id)
if alba_backend is None: # ALBA Backend with ID not found in model
continue
if alba_backend.abm_cluster is not None and len(
alba_backend.nsm_clusters
) > 0: # Clusters already exist
continue
# Create ABM Cluster
if alba_backend.abm_cluster is None:
abm_cluster = ABMCluster()
abm_cluster.name = cluster_name
abm_cluster.alba_backend = alba_backend
abm_cluster.config_location = ArakoonClusterConfig.CONFIG_KEY.format(
cluster_name)
abm_cluster.save()
else:
abm_cluster = alba_backend.abm_cluster
# Create ABM Services
abm_arakoon_config = cluster_arakoon_map[cluster_name]
abm_arakoon_config.pop('global')
arakoon_nodes = abm_arakoon_config.keys()
if internal is False:
services_to_create = 1
else:
if set(arakoon_nodes).difference(
set(storagerouter_map.keys())):
continue
services_to_create = len(arakoon_nodes)
for index in range(services_to_create):
service = Service()
service.name = 'arakoon-{0}-abm'.format(
alba_backend.name)
service.type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ALBA_MGR)
if internal is True:
arakoon_node_config = abm_arakoon_config[
arakoon_nodes[index]]
service.ports = [
arakoon_node_config['client_port'],
arakoon_node_config['messaging_port']
]
service.storagerouter = storagerouter_map[
arakoon_nodes[index]]
else:
service.ports = []
service.storagerouter = None
service.save()
abm_service = ABMService()
abm_service.service = service
abm_service.abm_cluster = abm_cluster
abm_service.save()
# Create NSM Clusters
for cluster_index, nsm_host in enumerate(
sorted(nsm_hosts,
key=lambda host: ExtensionsToolbox.
advanced_sort(host['cluster_id'], '_'))):
nsm_cluster_name = nsm_host['cluster_id']
nsm_arakoon_config = cluster_arakoon_map.get(
nsm_cluster_name)
if nsm_arakoon_config is None:
continue
number = cluster_index if internal is False else int(
nsm_cluster_name.split('_')[-1])
nsm_cluster = NSMCluster()
nsm_cluster.name = nsm_cluster_name
nsm_cluster.number = number
nsm_cluster.alba_backend = alba_backend
nsm_cluster.config_location = ArakoonClusterConfig.CONFIG_KEY.format(
nsm_cluster_name)
nsm_cluster.save()
# Create NSM Services
nsm_arakoon_config.pop('global')
arakoon_nodes = nsm_arakoon_config.keys()
if internal is False:
services_to_create = 1
else:
if set(arakoon_nodes).difference(
set(storagerouter_map.keys())):
continue
services_to_create = len(arakoon_nodes)
for service_index in range(services_to_create):
service = Service()
service.name = 'arakoon-{0}-nsm_{1}'.format(
alba_backend.name, number)
service.type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.NS_MGR)
if internal is True:
arakoon_node_config = nsm_arakoon_config[
arakoon_nodes[service_index]]
service.ports = [
arakoon_node_config['client_port'],
arakoon_node_config['messaging_port']
]
service.storagerouter = storagerouter_map[
arakoon_nodes[service_index]]
else:
service.ports = []
service.storagerouter = None
service.save()
nsm_service = NSMService()
nsm_service.service = service
nsm_service.nsm_cluster = nsm_cluster
nsm_service.save()
# Clean up all junction services no longer linked to an ALBA Backend
all_nsm_services = [
service.nsm_service for service in ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.NS_MGR).services
if service.nsm_service.nsm_cluster is None
]
all_abm_services = [
service.abm_service for service in ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ALBA_MGR).services
if service.abm_service.abm_cluster is None
]
for abm_service in all_abm_services:
abm_service.delete()
abm_service.service.delete()
for nsm_service in all_nsm_services:
nsm_service.delete()
nsm_service.service.delete()
################################
# Introduction of Active Drive #
################################
# Update slot_id and Alba Node relation for all OSDs
client = PersistentFactory.get_client()
disk_osd_map = {}
for key, data in client.prefix_entries('ovs_data_albaosd_'):
alba_disk_guid = data.get('alba_disk', {}).get('guid')
if alba_disk_guid is not None:
if alba_disk_guid not in disk_osd_map:
disk_osd_map[alba_disk_guid] = []
disk_osd_map[alba_disk_guid].append(
key.replace('ovs_data_albaosd_', ''))
try:
value = client.get(key)
value.pop('alba_disk', None)
client.set(key=key, value=value)
except Exception:
pass # We don't care if we would have any leftover AlbaDisk information in _data, but its cleaner not to
alba_guid_node_map = dict(
(an.guid, an) for an in AlbaNodeList.get_albanodes())
for key, data in client.prefix_entries('ovs_data_albadisk_'):
alba_disk_guid = key.replace('ovs_data_albadisk_', '')
alba_node_guid = data.get('alba_node', {}).get('guid')
if alba_disk_guid in disk_osd_map and alba_node_guid in alba_guid_node_map and len(
data.get('aliases', [])) > 0:
slot_id = data['aliases'][0].split('/')[-1]
for osd_guid in disk_osd_map[alba_disk_guid]:
try:
osd = AlbaOSD(osd_guid)
except ObjectNotFoundException:
continue
osd.slot_id = slot_id
osd.alba_node = alba_guid_node_map[alba_node_guid]
osd.save()
client.delete(key=key, must_exist=False)
# Remove unique constraints for AlbaNode IP
for key in client.prefix('ovs_unique_albanode_ip_'):
client.delete(key=key, must_exist=False)
# Remove relation for all Alba Disks
for key in client.prefix('ovs_reverseindex_albadisk_'):
client.delete(key=key, must_exist=False)
# Remove the relation between AlbaNode and AlbaDisk
for key in client.prefix('ovs_reverseindex_albanode_'):
if '|disks|' in key:
client.delete(key=key, must_exist=False)
return DALMigrator.THIS_VERSION
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
cls.factory = None
cls.initial_data = None
cls.persistent = PersistentFactory.get_client()
cls.persistent.clean()
cls.volatile = VolatileFactory.get_client()
cls.volatile.clean()
admin_group = Group()
admin_group.name = 'administrators'
admin_group.description = 'Administrators'
admin_group.save()
viewers_group = Group()
viewers_group.name = 'viewers'
viewers_group.description = 'Viewers'
viewers_group.save()
# Create users
admin = User()
admin.username = '******'
admin.password = hashlib.sha256('admin').hexdigest()
admin.is_active = True
admin.group = admin_group
admin.save()
admin_npg = User()
admin_npg.username = '******'
admin_npg.password = hashlib.sha256('admin_npg').hexdigest()
admin_npg.is_active = True
admin_npg.group = admin_group
admin_npg.save()
admin_na = User()
admin_na.username = '******'
admin_na.password = hashlib.sha256('admin_na').hexdigest()
admin_na.is_active = False
admin_na.group = admin_group
admin_na.save()
user = User()
user.username = '******'
user.password = hashlib.sha256('user').hexdigest()
user.is_active = True
user.group = viewers_group
user.save()
sort_combinations = [('bb', 'aa'), ('aa', 'cc'), ('bb', 'dd'), ('aa', 'bb')] # No logical ordering
for username, password in sort_combinations:
sort_user = User()
sort_user.username = username
sort_user.password = password
sort_user.is_active = True
sort_user.group = viewers_group
sort_user.save()
# Create internal OAuth 2 clients
admin_client = Client()
admin_client.ovs_type = 'INTERNAL'
admin_client.grant_type = 'PASSWORD'
admin_client.user = admin
admin_client.save()
admin_na_client = Client()
admin_na_client.ovs_type = 'INTERNAL'
admin_na_client.grant_type = 'PASSWORD'
admin_na_client.user = admin_na
admin_na_client.save()
user_client = Client()
user_client.ovs_type = 'INTERNAL'
user_client.grant_type = 'PASSWORD'
user_client.user = user
user_client.save()
# Create roles
read_role = Role()
read_role.code = 'read'
read_role.name = 'Read'
read_role.description = 'Can read objects'
read_role.save()
write_role = Role()
write_role.code = 'write'
write_role.name = 'Write'
write_role.description = 'Can write objects'
write_role.save()
manage_role = Role()
manage_role.code = 'manage'
manage_role.name = 'Manage'
manage_role.description = 'Can manage the system'
manage_role.save()
# Attach groups to roles
mapping = [
(admin_group, [read_role, write_role, manage_role]),
(viewers_group, [read_role])
]
for setting in mapping:
for role in setting[1]:
rolegroup = RoleGroup()
rolegroup.group = setting[0]
rolegroup.role = role
rolegroup.save()
for user in setting[0].users:
for role in setting[1]:
for client in user.clients:
roleclient = RoleClient()
roleclient.client = client
roleclient.role = role
roleclient.save()
cls.initial_data = PersistentFactory.store._read(), VolatileFactory.store._read()
sys.modules['backend.serializers.serializers'] = Serializers
sys.path.append('/opt/OpenvStorage')
sys.path.append('/opt/OpenvStorage/webapps')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
from django.conf import settings
settings.VERSION = (1, 2, 3)
from django.test import RequestFactory
cls.factory = RequestFactory()
fakesleep.monkey_patch()
def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None):
"""
Loads an object with a given guid. If no guid is given, a new object
is generated with a new guid.
* guid: The guid indicating which object should be loaded
* datastoreWins: Optional boolean indicating save conflict resolve management.
** True: when saving, external modified fields will not be saved
** False: when saving, all changed data will be saved, regardless of external updates
** None: in case changed field were also changed externally, an error will be raised
"""
# Initialize super class
super(DataObject, self).__init__()
# Initialize internal fields
self._frozen = False
self._datastore_wins = datastore_wins
self._guid = None # Guid identifier of the object
self._original = {} # Original data copy
self._metadata = {} # Some metadata, mainly used for unit testing
self._data = {} # Internal data storage
self._objects = {} # Internal objects storage
# Initialize public fields
self.dirty = False
self.volatile = volatile
# Worker fields/objects
self._classname = self.__class__.__name__.lower()
# Rebuild _relation types
hybrid_structure = HybridRunner.get_hybrids()
for relation in self._relations:
if relation.foreign_type is not None:
identifier = Descriptor(relation.foreign_type).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object()
# Init guid
self._new = False
if guid is None:
self._guid = str(uuid.uuid4())
self._new = True
else:
self._guid = str(guid)
# Build base keys
self._key = '{0}_{1}_{2}'.format(DataObject.NAMESPACE, self._classname, self._guid)
# Worker mutexes
self._mutex_version = volatile_mutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid))
# Load data from cache or persistent backend where appropriate
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._metadata['cache'] = None
if self._new:
self._data = {}
else:
if data is not None:
self._data = copy.deepcopy(data)
self._metadata['cache'] = None
else:
self._data = self._volatile.get(self._key)
if self._data is None:
self._metadata['cache'] = False
try:
self._data = self._persistent.get(self._key)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
else:
self._metadata['cache'] = True
# Set default values on new fields
for prop in self._properties:
if prop.name not in self._data:
self._data[prop.name] = prop.default
self._add_property(prop)
# Load relations
for relation in self._relations:
if relation.name not in self._data:
if relation.foreign_type is None:
cls = self.__class__
else:
cls = relation.foreign_type
self._data[relation.name] = Descriptor(cls).descriptor
self._add_relation_property(relation)
# Add wrapped properties
for dynamic in self._dynamics:
self._add_dynamic_property(dynamic)
# Load foreign keys
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
self._objects[key] = {'info': info,
'data': None}
self._add_list_property(key, info['list'])
if _hook is not None and hasattr(_hook, '__call__'):
_hook()
if not self._new:
# Re-cache the object, if required
if self._metadata['cache'] is False:
# The data wasn't loaded from the cache, so caching is required now
try:
self._mutex_version.acquire(30)
this_version = self._data['_version']
store_version = self._persistent.get(self._key)['_version']
if this_version == store_version:
self._volatile.set(self._key, self._data)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
finally:
self._mutex_version.release()
# Freeze property creation
self._frozen = True
# Optionally, initialize some fields
if data is not None:
for prop in self._properties:
if prop.name in data:
setattr(self, prop.name, data[prop.name])
# Store original data
self._original = copy.deepcopy(self._data)
def new_function(*args, **kwargs):
"""
Wrapped function
:param args: Arguments without default values
:param kwargs: Arguments with default values
"""
def log_message(message, level='info'):
"""
Log a message with some additional information
:param message: Message to log
:param level: Log level
:return: None
"""
if level not in ('info', 'warning', 'debug', 'error'):
raise ValueError(
'Unsupported log level "{0}" specified'.format(level))
complete_message = 'Ensure single {0} mode - ID {1} - {2}'.format(
mode, now, message)
getattr(logger, level)(complete_message)
def update_value(key, append, value_to_store=None):
"""
Store the specified value in the PersistentFactory
:param key: Key to store the value for
:param append: If True, the specified value will be appended else element at index 0 will be popped
:param value_to_store: Value to append to the list
:return: Updated value
"""
with VolatileMutex(name=key, wait=5):
if persistent_client.exists(key):
val = persistent_client.get(key)
if append is True and value_to_store is not None:
val['values'].append(value_to_store)
elif append is False and len(val['values']) > 0:
val['values'].pop(0)
else:
log_message('Setting initial value for key {0}'.format(
persistent_key))
val = {'mode': mode, 'values': []}
persistent_client.set(key, val)
return val
now = '{0}_{1}'.format(
int(time.time()), ''.join(
random.choice(string.ascii_letters + string.digits)
for _ in range(10)))
task_names = [
task_name
] if extra_task_names is None else [task_name] + extra_task_names
persistent_key = '{0}_{1}'.format(ENSURE_SINGLE_KEY, task_name)
persistent_client = PersistentFactory.get_client()
if mode == 'DEFAULT':
with VolatileMutex(persistent_key, wait=5):
for task in task_names:
key_to_check = '{0}_{1}'.format(
ENSURE_SINGLE_KEY, task)
if persistent_client.exists(key_to_check):
log_message(
'Execution of task {0} discarded'.format(
task_name))
return None
log_message('Setting key {0}'.format(persistent_key))
persistent_client.set(persistent_key, {'mode': mode})
try:
output = function(*args, **kwargs)
log_message(
'Task {0} finished successfully'.format(task_name))
return output
finally:
with VolatileMutex(persistent_key, wait=5):
if persistent_client.exists(persistent_key):
log_message(
'Deleting key {0}'.format(persistent_key))
persistent_client.delete(persistent_key)
elif mode == 'CHAINED':
if extra_task_names is not None:
log_message('Extra tasks are not allowed in this mode',
level='error')
raise ValueError(
'Ensure single {0} mode - ID {1} - Extra tasks are not allowed in this mode'
.format(mode, now))
# 1. Create key to be stored in arakoon and update kwargs with args
timeout = kwargs.pop(
'chain_timeout'
) if 'chain_timeout' in kwargs else global_timeout
function_info = inspect.getargspec(function)
kwargs_dict = {}
for index, arg in enumerate(args):
kwargs_dict[function_info.args[index]] = arg
kwargs_dict.update(kwargs)
params_info = 'with params {0}'.format(
kwargs_dict) if kwargs_dict else 'with default params'
# 2. Set the key in arakoon if non-existent
value = update_value(key=persistent_key, append=True)
# 3. Validate whether another job with same params is being executed, skip if so
for item in value['values'][
1:]: # 1st element is processing job, we check all other queued jobs for identical params
if item['kwargs'] == kwargs_dict:
log_message(
'Execution of task {0} {1} discarded because of identical parameters'
.format(task_name, params_info))
return None
log_message('New task {0} {1} scheduled for execution'.format(
task_name, params_info))
update_value(key=persistent_key,
append=True,
value_to_store={
'kwargs': kwargs_dict,
'timestamp': now
})
# 4. Poll the arakoon to see whether this call is the first in list, if so --> execute, else wait
first_element = None
counter = 0
while first_element != now and counter < timeout:
if persistent_client.exists(persistent_key):
value = persistent_client.get(persistent_key)
first_element = value['values'][0]['timestamp']
if first_element == now:
try:
if counter != 0:
current_time = int(time.time())
starting_time = int(now.split('_')[0])
log_message(
'Task {0} {1} had to wait {2} seconds before being able to start'
.format(task_name, params_info,
current_time - starting_time))
output = function(*args, **kwargs)
log_message(
'Task {0} finished successfully'.format(
task_name))
finally:
update_value(key=persistent_key, append=False)
return output
counter += 1
time.sleep(1)
if counter == timeout:
update_value(key=persistent_key, append=False)
log_message(
'Could not start task {0} {1}, within expected time ({2}s). Removed it from queue'
.format(task_name, params_info, timeout),
level='error')
raise EnsureSingleTimeoutReached(
'Ensure single {0} mode - ID {1} - Task {2} could not be started within timeout of {3}s'
.format(mode, now, task_name, timeout))
else:
raise ValueError(
'Unsupported mode "{0}" provided'.format(mode))
def migrate(previous_version):
"""
Migrates from any version to any version, running all migrations required
If previous_version is for example 0 and this script is at
version 3 it will execute two steps:
- 1 > 2
- 2 > 3
@param previous_version: The previous version from which to start the migration.
"""
working_version = previous_version
# Version 1 introduced:
# - The datastore is still empty, add defaults
if working_version < 1:
from ovs.dal.hybrids.user import User
from ovs.dal.hybrids.group import Group
from ovs.dal.hybrids.role import Role
from ovs.dal.hybrids.client import Client
from ovs.dal.hybrids.failuredomain import FailureDomain
from ovs.dal.hybrids.j_rolegroup import RoleGroup
from ovs.dal.hybrids.j_roleclient import RoleClient
from ovs.dal.hybrids.backendtype import BackendType
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.hybrids.branding import Branding
from ovs.dal.lists.backendtypelist import BackendTypeList
# Create groups
admin_group = Group()
admin_group.name = 'administrators'
admin_group.description = 'Administrators'
admin_group.save()
viewers_group = Group()
viewers_group.name = 'viewers'
viewers_group.description = 'Viewers'
viewers_group.save()
# Create users
admin = User()
admin.username = '******'
admin.password = hashlib.sha256('admin').hexdigest()
admin.is_active = True
admin.group = admin_group
admin.save()
# Create internal OAuth 2 clients
admin_pw_client = Client()
admin_pw_client.ovs_type = 'INTERNAL'
admin_pw_client.grant_type = 'PASSWORD'
admin_pw_client.user = admin
admin_pw_client.save()
admin_cc_client = Client()
admin_cc_client.ovs_type = 'INTERNAL'
admin_cc_client.grant_type = 'CLIENT_CREDENTIALS'
admin_cc_client.client_secret = ''.join(random.choice(string.ascii_letters +
string.digits +
'|_=+*#@!/-[]{}<>.?,\'";:~')
for _ in range(128))
admin_cc_client.user = admin
admin_cc_client.save()
# Create roles
read_role = Role()
read_role.code = 'read'
read_role.name = 'Read'
read_role.description = 'Can read objects'
read_role.save()
write_role = Role()
write_role.code = 'write'
write_role.name = 'Write'
write_role.description = 'Can write objects'
write_role.save()
manage_role = Role()
manage_role.code = 'manage'
manage_role.name = 'Manage'
manage_role.description = 'Can manage the system'
manage_role.save()
# Attach groups to roles
mapping = [
(admin_group, [read_role, write_role, manage_role]),
(viewers_group, [read_role])
]
for setting in mapping:
for role in setting[1]:
rolegroup = RoleGroup()
rolegroup.group = setting[0]
rolegroup.role = role
rolegroup.save()
for user in setting[0].users:
for role in setting[1]:
for client in user.clients:
roleclient = RoleClient()
roleclient.client = client
roleclient.role = role
roleclient.save()
# Add backends
for backend_type_info in [('Ceph', 'ceph_s3'), ('Amazon', 'amazon_s3'), ('Swift', 'swift_s3'),
('Local', 'local'), ('Distributed', 'distributed'), ('ALBA', 'alba')]:
code = backend_type_info[1]
backend_type = BackendTypeList.get_backend_type_by_code(code)
if backend_type is None:
backend_type = BackendType()
backend_type.name = backend_type_info[0]
backend_type.code = code
backend_type.save()
# Add service types
for service_type_info in [ServiceType.SERVICE_TYPES.MD_SERVER, ServiceType.SERVICE_TYPES.ALBA_PROXY, ServiceType.SERVICE_TYPES.ARAKOON]:
service_type = ServiceType()
service_type.name = service_type_info
service_type.save()
# Branding
branding = Branding()
branding.name = 'Default'
branding.description = 'Default bootstrap theme'
branding.css = 'bootstrap-default.min.css'
branding.productname = 'Open vStorage'
branding.is_default = True
branding.save()
slate = Branding()
slate.name = 'Slate'
slate.description = 'Dark bootstrap theme'
slate.css = 'bootstrap-slate.min.css'
slate.productname = 'Open vStorage'
slate.is_default = False
slate.save()
# Failure Domain
failure_domain = FailureDomain()
failure_domain.name = 'Default'
failure_domain.save()
# We're now at version 1
working_version = 1
# Version 2 introduced:
# - new Descriptor format
if working_version < 2:
import imp
from ovs.dal.helpers import Descriptor
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
keys = client.prefix('ovs_data')
for key in keys:
data = client.get(key)
modified = False
for entry in data.keys():
if isinstance(data[entry], dict) and 'source' in data[entry] and 'hybrids' in data[entry]['source']:
filename = data[entry]['source']
if not filename.startswith('/'):
filename = '/opt/OpenvStorage/ovs/dal/{0}'.format(filename)
module = imp.load_source(data[entry]['name'], filename)
cls = getattr(module, data[entry]['type'])
new_data = Descriptor(cls, cached=False).descriptor
if 'guid' in data[entry]:
new_data['guid'] = data[entry]['guid']
data[entry] = new_data
modified = True
if modified is True:
data['_version'] += 1
client.set(key, data)
# We're now at version 2
working_version = 2
# Version 3 introduced:
# - new Descriptor format
if working_version < 3:
import imp
from ovs.dal.helpers import Descriptor
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
keys = client.prefix('ovs_data')
for key in keys:
data = client.get(key)
modified = False
for entry in data.keys():
if isinstance(data[entry], dict) and 'source' in data[entry]:
module = imp.load_source(data[entry]['name'], data[entry]['source'])
cls = getattr(module, data[entry]['type'])
new_data = Descriptor(cls, cached=False).descriptor
if 'guid' in data[entry]:
new_data['guid'] = data[entry]['guid']
data[entry] = new_data
modified = True
if modified is True:
data['_version'] += 1
client.set(key, data)
working_version = 3
# Version 4 introduced:
# - Flexible SSD layout
if working_version < 4:
import os
from ovs.dal.hybrids.diskpartition import DiskPartition
from ovs.dal.hybrids.j_storagedriverpartition import StorageDriverPartition
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.extensions.generic.remote import remote
from ovs.extensions.generic.sshclient import SSHClient
from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration
for service in ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.MD_SERVER).services:
mds_service = service.mds_service
storagedriver = None
for current_storagedriver in service.storagerouter.storagedrivers:
if current_storagedriver.vpool_guid == mds_service.vpool_guid:
storagedriver = current_storagedriver
break
tasks = {}
if storagedriver._data.get('mountpoint_md'):
tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_md'),
storagedriver.vpool.name,
mds_service.number)] = (DiskPartition.ROLES.DB, StorageDriverPartition.SUBROLE.MDS)
if storagedriver._data.get('mountpoint_temp'):
tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_temp'),
storagedriver.vpool.name,
mds_service.number)] = (DiskPartition.ROLES.SCRUB, StorageDriverPartition.SUBROLE.MDS)
for disk in service.storagerouter.disks:
for partition in disk.partitions:
for directory, (role, subrole) in tasks.iteritems():
with remote(storagedriver.storagerouter.ip, [os], username='******') as rem:
stat_dir = directory
while not rem.os.path.exists(stat_dir) and stat_dir != '/':
stat_dir = stat_dir.rsplit('/', 1)[0]
if not stat_dir:
stat_dir = '/'
inode = rem.os.stat(stat_dir).st_dev
if partition.inode == inode:
if role not in partition.roles:
partition.roles.append(role)
partition.save()
number = 0
migrated = False
for sd_partition in storagedriver.partitions:
if sd_partition.role == role and sd_partition.sub_role == subrole:
if sd_partition.mds_service == mds_service:
migrated = True
break
if sd_partition.partition_guid == partition.guid:
number = max(sd_partition.number, number)
if migrated is False:
sd_partition = StorageDriverPartition()
sd_partition.role = role
sd_partition.sub_role = subrole
sd_partition.partition = partition
sd_partition.storagedriver = storagedriver
sd_partition.mds_service = mds_service
sd_partition.size = None
sd_partition.number = number + 1
sd_partition.save()
client = SSHClient(storagedriver.storagerouter, username='******')
path = sd_partition.path.rsplit('/', 1)[0]
if path:
client.dir_create(path)
client.dir_chown(path, 'ovs', 'ovs')
client.dir_create(directory)
client.dir_chown(directory, 'ovs', 'ovs')
client.symlink({sd_partition.path: directory})
for storagedriver in StorageDriverList.get_storagedrivers():
migrated_objects = {}
for disk in storagedriver.storagerouter.disks:
for partition in disk.partitions:
# Process all mountpoints that are unique and don't have a specified size
for key, (role, sr_info) in {'mountpoint_md': (DiskPartition.ROLES.DB, {'metadata_{0}': StorageDriverPartition.SUBROLE.MD,
'tlogs_{0}': StorageDriverPartition.SUBROLE.TLOG}),
'mountpoint_fragmentcache': (DiskPartition.ROLES.WRITE, {'fcache_{0}': StorageDriverPartition.SUBROLE.FCACHE}),
'mountpoint_foc': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD,
'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}),
'mountpoint_dtl': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD,
'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}),
'mountpoint_readcaches': (DiskPartition.ROLES.READ, {'': None}),
'mountpoint_writecaches': (DiskPartition.ROLES.WRITE, {'sco_{0}': StorageDriverPartition.SUBROLE.SCO})}.iteritems():
if key in storagedriver._data:
is_list = isinstance(storagedriver._data[key], list)
entries = storagedriver._data[key][:] if is_list is True else [storagedriver._data[key]]
for entry in entries:
if not entry:
if is_list:
storagedriver._data[key].remove(entry)
if len(storagedriver._data[key]) == 0:
del storagedriver._data[key]
else:
del storagedriver._data[key]
else:
with remote(storagedriver.storagerouter.ip, [os], username='******') as rem:
inode = rem.os.stat(entry).st_dev
if partition.inode == inode:
if role not in partition.roles:
partition.roles.append(role)
partition.save()
for folder, subrole in sr_info.iteritems():
number = 0
migrated = False
for sd_partition in storagedriver.partitions:
if sd_partition.role == role and sd_partition.sub_role == subrole:
if sd_partition.partition_guid == partition.guid:
number = max(sd_partition.number, number)
if migrated is False:
sd_partition = StorageDriverPartition()
sd_partition.role = role
sd_partition.sub_role = subrole
sd_partition.partition = partition
sd_partition.storagedriver = storagedriver
sd_partition.size = None
sd_partition.number = number + 1
sd_partition.save()
if folder:
source = '{0}/{1}'.format(entry, folder.format(storagedriver.vpool.name))
else:
source = entry
client = SSHClient(storagedriver.storagerouter, username='******')
path = sd_partition.path.rsplit('/', 1)[0]
if path:
client.dir_create(path)
client.dir_chown(path, 'ovs', 'ovs')
client.symlink({sd_partition.path: source})
migrated_objects[source] = sd_partition
if is_list:
storagedriver._data[key].remove(entry)
if len(storagedriver._data[key]) == 0:
del storagedriver._data[key]
else:
del storagedriver._data[key]
storagedriver.save()
if 'mountpoint_bfs' in storagedriver._data:
storagedriver.mountpoint_dfs = storagedriver._data['mountpoint_bfs']
if not storagedriver.mountpoint_dfs:
storagedriver.mountpoint_dfs = None
del storagedriver._data['mountpoint_bfs']
storagedriver.save()
if 'mountpoint_temp' in storagedriver._data:
del storagedriver._data['mountpoint_temp']
storagedriver.save()
if migrated_objects:
print 'Loading sizes'
config = StorageDriverConfiguration('storagedriver', storagedriver.vpool_guid, storagedriver.storagedriver_id)
config.load()
for readcache in config.configuration.get('content_addressed_cache', {}).get('clustercache_mount_points', []):
path = readcache.get('path', '').rsplit('/', 1)[0]
size = int(readcache['size'].strip('KiB')) * 1024 if 'size' in readcache else None
if path in migrated_objects:
migrated_objects[path].size = long(size)
migrated_objects[path].save()
for writecache in config.configuration.get('scocache', {}).get('scocache_mount_points', []):
path = writecache.get('path', '')
size = int(writecache['size'].strip('KiB')) * 1024 if 'size' in writecache else None
if path in migrated_objects:
migrated_objects[path].size = long(size)
migrated_objects[path].save()
working_version = 4
# Version 5 introduced:
# - Failure Domains
if working_version < 5:
import os
from ovs.dal.hybrids.failuredomain import FailureDomain
from ovs.dal.lists.failuredomainlist import FailureDomainList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.extensions.generic.remote import remote
from ovs.extensions.generic.sshclient import SSHClient
failure_domains = FailureDomainList.get_failure_domains()
if len(failure_domains) > 0:
failure_domain = failure_domains[0]
else:
failure_domain = FailureDomain()
failure_domain.name = 'Default'
failure_domain.save()
for storagerouter in StorageRouterList.get_storagerouters():
change = False
if storagerouter.primary_failure_domain is None:
storagerouter.primary_failure_domain = failure_domain
change = True
if storagerouter.rdma_capable is None:
client = SSHClient(storagerouter, username='******')
rdma_capable = False
with remote(client.ip, [os], username='******') as rem:
for root, dirs, files in rem.os.walk('/sys/class/infiniband'):
for directory in dirs:
ports_dir = '/'.join([root, directory, 'ports'])
if not rem.os.path.exists(ports_dir):
continue
for sub_root, sub_dirs, _ in rem.os.walk(ports_dir):
if sub_root != ports_dir:
continue
for sub_directory in sub_dirs:
state_file = '/'.join([sub_root, sub_directory, 'state'])
if rem.os.path.exists(state_file):
if 'ACTIVE' in client.run('cat {0}'.format(state_file)):
rdma_capable = True
storagerouter.rdma_capable = rdma_capable
change = True
if change is True:
storagerouter.save()
working_version = 5
# Version 6 introduced:
# - Distributed scrubbing
if working_version < 6:
from ovs.dal.hybrids.diskpartition import DiskPartition
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.extensions.generic.sshclient import SSHClient
for storage_driver in StorageDriverList.get_storagedrivers():
root_client = SSHClient(storage_driver.storagerouter, username='******')
for partition in storage_driver.partitions:
if partition.role == DiskPartition.ROLES.SCRUB:
old_path = partition.path
partition.sub_role = None
partition.save()
partition.invalidate_dynamics(['folder', 'path'])
if root_client.dir_exists(partition.path):
continue # New directory already exists
if '_mds_' in old_path:
if root_client.dir_exists(old_path):
root_client.symlink({partition.path: old_path})
if not root_client.dir_exists(partition.path):
root_client.dir_create(partition.path)
root_client.dir_chmod(partition.path, 0777)
working_version = 6
# Version 7 introduced:
# - vPool status
if working_version < 7:
from ovs.dal.hybrids import vpool
reload(vpool)
from ovs.dal.hybrids.vpool import VPool
from ovs.dal.lists.vpoollist import VPoolList
for _vpool in VPoolList.get_vpools():
vpool = VPool(_vpool.guid)
if hasattr(vpool, 'status') and vpool.status is None:
vpool.status = VPool.STATUSES.RUNNING
vpool.save()
working_version = 7
# Version 10 introduced:
# - Reverse indexes are stored in persistent store
# - Store more non-changing metadata on disk iso using a dynamic property
if working_version < 10:
from ovs.dal.helpers import HybridRunner, Descriptor
from ovs.dal.datalist import DataList
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.extensions.storage.volatilefactory import VolatileFactory
persistent = PersistentFactory.get_client()
for prefix in ['ovs_listcache', 'ovs_reverseindex']:
for key in persistent.prefix(prefix):
persistent.delete(key)
for key in persistent.prefix('ovs_data_'):
persistent.set(key, persistent.get(key))
base_reverse_key = 'ovs_reverseindex_{0}_{1}|{2}|{3}'
hybrid_structure = HybridRunner.get_hybrids()
for class_descriptor in hybrid_structure.values():
cls = Descriptor().load(class_descriptor).get_object()
all_objects = DataList(cls, {'type': DataList.where_operator.AND,
'items': []})
for item in all_objects:
guid = item.guid
for relation in item._relations:
if relation.foreign_type is None:
rcls = cls
rclsname = rcls.__name__.lower()
else:
rcls = relation.foreign_type
rclsname = rcls.__name__.lower()
key = relation.name
rguid = item._data[key]['guid']
if rguid is not None:
reverse_key = base_reverse_key.format(rclsname, rguid, relation.foreign_key, guid)
persistent.set(reverse_key, 0)
volatile = VolatileFactory.get_client()
try:
volatile._client.flush_all()
except:
pass
from ovs.dal.lists.vdisklist import VDiskList
for vdisk in VDiskList.get_vdisks():
try:
vdisk.metadata = {'lba_size': vdisk.info['lba_size'],
'cluster_multiplier': vdisk.info['cluster_multiplier']}
vdisk.save()
except:
pass
working_version = 10
# Version 11 introduced:
# - ALBA accelerated ALBA, meaning different vpool.metadata information
if working_version < 11:
from ovs.dal.lists.vpoollist import VPoolList
for vpool in VPoolList.get_vpools():
vpool.metadata = {'backend': vpool.metadata}
if 'metadata' in vpool.metadata['backend']:
vpool.metadata['backend']['arakoon_config'] = vpool.metadata['backend'].pop('metadata')
if 'backend_info' in vpool.metadata['backend']:
vpool.metadata['backend']['backend_info']['fragment_cache_on_read'] = True
vpool.metadata['backend']['backend_info']['fragment_cache_on_write'] = False
vpool.save()
working_version = 11
return working_version
