def get_client(client_type=None):
"""
Returns a volatile storage client
"""
if not hasattr(VolatileFactory, 'store') or VolatileFactory.store is None:
if client_type is None:
client_type = Configuration.get('ovs.core.storage.volatile')
VolatileFactory.store = None
if client_type == 'memcache':
from ovs.extensions.storage.volatile.memcachestore import MemcacheStore
memcache_servers = list()
memcache_config = RawConfigParser()
memcache_config.read(os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg'))
nodes = [node.strip() for node in memcache_config.get('main', 'nodes').split(',')]
nodes.sort()
for node in nodes:
location = memcache_config.get(node, 'location')
memcache_servers.append(location)
VolatileFactory.store = MemcacheStore(memcache_servers)
if client_type == 'default':
from ovs.extensions.storage.volatile.dummystore import DummyVolatileStore
VolatileFactory.store = DummyVolatileStore()
if VolatileFactory.store is None:
raise RuntimeError('Invalid client_type specified')
return VolatileFactory.store
def get_client(client_type=None):
"""
Returns a volatile storage client
"""
if not hasattr(VolatileFactory,
'store') or VolatileFactory.store is None:
if client_type is None:
client_type = Configuration.get('ovs.core.storage.volatile')
VolatileFactory.store = None
if client_type == 'memcache':
from ovs.extensions.storage.volatile.memcachestore import MemcacheStore
memcache_servers = list()
memcache_config = ConfigObj(
os.path.join(Configuration.get('ovs.core.cfgdir'),
'memcacheclient.cfg'))
nodes = memcache_config.get('main')['nodes'] if type(
memcache_config.get('main')['nodes']) == list else [
memcache_config.get('main')['nodes']
]
nodes = [node.strip() for node in nodes]
nodes.sort()
for node in nodes:
location = memcache_config.get(node)['location']
memcache_servers.append(location)
VolatileFactory.store = MemcacheStore(memcache_servers)
if client_type == 'default':
from ovs.extensions.storage.volatile.dummystore import DummyVolatileStore
VolatileFactory.store = DummyVolatileStore()
if VolatileFactory.store is None:
raise RuntimeError('Invalid client_type specified')
return VolatileFactory.store
def get_physical_metadata(files, storagerouter_guid):
"""
Gets physical information about the machine this task is running on
"""
from ovs.lib.vpool import VPoolController
storagerouter = StorageRouter(storagerouter_guid)
mountpoints = check_output('mount -v', shell=True).strip().split('\n')
mountpoints = [p.split(' ')[2] for p in mountpoints if len(p.split(' ')) > 2
and not p.split(' ')[2].startswith('/dev') and not p.split(' ')[2].startswith('/proc')
and not p.split(' ')[2].startswith('/sys') and not p.split(' ')[2].startswith('/run')
and p.split(' ')[2] != '/']
arakoon_mountpoint = Configuration.get('ovs.core.db.arakoon.location')
if arakoon_mountpoint in mountpoints:
mountpoints.remove(arakoon_mountpoint)
if storagerouter.pmachine.hvtype == 'KVM':
ipaddresses = ['127.0.0.1']
else:
ip_path = Configuration.get('ovs.core.ip.path')
if ip_path is None:
ip_path = "`which ip`"
ipaddresses = check_output("{0} a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1".format(ip_path), shell=True).strip().split('\n')
ipaddresses = [ip.strip() for ip in ipaddresses]
ipaddresses.remove('127.0.0.1')
allow_vpool = VPoolController.can_be_served_on(storagerouter_guid)
file_existence = {}
for check_file in files:
file_existence[check_file] = os.path.exists(check_file) and os.path.isfile(check_file)
return {'mountpoints': mountpoints,
'ipaddresses': ipaddresses,
'files': file_existence,
'allow_vpool': allow_vpool}
def __init__(self):
self._config_corefile = os.path.join(
Configuration.get('ovs.core.cfgdir'), 'templates',
'ganesha-core.conf')
self._config_exportfile = os.path.join(
Configuration.get('ovs.core.cfgdir'), 'templates',
'ganesha-export.conf')
def __init__(self):
"""
Initializes the client
"""
self._endpoint = Configuration.get('ovs.support.endpoint')
self._api = Configuration.get('ovs.support.api')
self._enable_support = int(Configuration.get('ovs.support.enablesupport')) > 0
self.interval = int(Configuration.get('ovs.support.interval'))
self._url = 'https://{0}/{1}'.format(self._endpoint, self._api)
def __init__(self, vpool_name):
self._vpool = vpool_name
self._config_specfile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'templates', 'volumedriverfs.json')
if not os.path.exists(os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools')):
os.makedirs(os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools'))
self._config_file = os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools', '{}.json'.format(vpool_name))
self._config_tmpfile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools', '{}.json.tmp'.format(vpool_name))
self._config_readfile_handler = None
self._config_file_handler = None
self._config_specfile_handler = None
self._config_file_content = None
def setUpClass(cls):
ArakoonInstaller.ARAKOON_CONFIG_DIR = '/tmp/cfg'
ArakoonInstaller.ARAKOON_CONFIG_FILE = '/tmp/cfg/{0}/{0}.cfg'
TestArakoonInstaller.expected_global = '[global]\ncluster_id = {0}\ncluster = {1}\nplugins = \n\n'
TestArakoonInstaller.expected_base = '[{0}]\nname = {0}\nip = {1}\nclient_port = {2}\nmessaging_port = {3}\ntlog_compression = snappy\nlog_level = info\nlog_dir = /var/log/arakoon/one\nhome = /tmp/db/arakoon/one\ntlog_dir = /tmp/db/tlogs/one\nfsync = true\n\n'
# SSH client
def _load(ip, password=None):
_client = SSHClient._load(ip, password)
_client._ip = ip
return _client
SSHClient._load = staticmethod(SSHClient.load)
SSHClient.load = staticmethod(_load)
# System
def _get_my_machine_id(_client):
return TestArakoonInstaller.nodes[_client._ip]
def _read_remote_config(_client, _key):
return Configuration.get(_key)
System.get_my_machine_id = staticmethod(_get_my_machine_id)
System.read_remote_config = staticmethod(_read_remote_config)
# Configuration
def _get(key):
if key == 'ovs.core.storage.persistent':
return 'arakoon'
if key == 'ovs.core.db.arakoon.clusterid':
return 'ovsdb'
c = PersistentFactory.get_client()
if c.exists(key):
return c.get(key)
return None
def _get_int(key):
return int(Configuration.get(key))
def _set(key, value):
c = PersistentFactory.get_client()
c.set(key, value)
Configuration.get = staticmethod(_get)
Configuration.getInt = staticmethod(_get_int)
Configuration.set = staticmethod(_set)
Configuration.set('ovs.ports.arakoon', 22000)
Configuration.set('ovs.core.db.arakoon.location', '/tmp/db')
def get_heartbeat_data():
"""
Returns heartbeat data
"""
data = {'cid': Configuration.get('ovs.support.cid'),
'nid': Configuration.get('ovs.support.nid'),
'metadata': {},
'errors': []}
try:
# Versions
data['metadata']['versions'] = Package.get_versions()
except Exception, ex:
data['errors'].append(str(ex))
def get_mds_storagedriver_config_set(vpool):
"""
Builds a configuration for all StorageRouters from a given VPool with following goals:
* Primary MDS is the local one
* All slaves are on different hosts
* Maximum `mds.safety` nodes are returned
"""
mds_per_storagerouter = {}
mds_per_load = {}
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
mds_service, load = MDSServiceController.get_preferred_mds(storagerouter, vpool, include_load=True)
mds_per_storagerouter[storagerouter.guid] = {'host': storagerouter.ip, 'port': mds_service.service.ports[0]}
if load not in mds_per_load:
mds_per_load[load] = []
mds_per_load[load].append(storagerouter.guid)
safety = Configuration.getInt('ovs.storagedriver.mds.safety')
config_set = {}
for storagerouter_guid in mds_per_storagerouter:
config_set[storagerouter_guid] = [mds_per_storagerouter[storagerouter_guid]]
for load in sorted(mds_per_load.keys()):
if len(config_set[storagerouter_guid]) >= safety:
break
sr_guids = mds_per_load[load]
random.shuffle(sr_guids)
for sr_guid in sr_guids:
if len(config_set[storagerouter_guid]) >= safety:
break
if sr_guid != storagerouter_guid:
config_set[storagerouter_guid].append(mds_per_storagerouter[sr_guid])
return config_set
def __init__(self, config_type, vpool_name, number=None):
"""
Initializes the class
"""
def make_configure(sct):
""" section closure """
return lambda **kwargs: self._add(sct, **kwargs)
if config_type not in ['storagedriver', 'metadataserver']:
raise RuntimeError('Invalid configuration type. Allowed: storagedriver, metadataserver')
self.config_type = config_type
self.vpool_name = vpool_name
self.configuration = {}
self.is_new = True
self.number = number
self.params = copy.deepcopy(StorageDriverConfiguration.parameters) # Never use parameters directly
self.base_path = '{0}/storagedriver/{1}'.format(Configuration.get('ovs.core.cfgdir'), self.config_type)
if self.number is None:
self.path = '{0}/{1}.json'.format(self.base_path, self.vpool_name)
else:
self.path = '{0}/{1}_{2}.json'.format(self.base_path, self.vpool_name, self.number)
# Fix some manual "I know what I'm doing" overrides
backend_connection_manager = 'backend_connection_manager'
self.params[self.config_type][backend_connection_manager]['optional'].append('s3_connection_strict_consistency')
# Generate configure_* methods
for section in self.params[self.config_type]:
setattr(self, 'configure_{0}'.format(section), make_configure(section))
def run_event_consumer():
"""
Check whether to run the event consumer
"""
rmq_config = RawConfigParser()
rmq_config.read(os.path.join(Configuration.get("ovs.core.cfgdir"), "rabbitmqclient.cfg"))
machine_id = System.get_my_machine_id()
return rmq_config.has_section(machine_id)
def _has_plugin(self):
"""
Checks whether this BackendType has a plugin installed
"""
try:
return True if Configuration.get('ovs.plugins.backend.' + self.code) else False
except:
return False
def _get_memcache_nodes():
"""
Get the memcache nodes
"""
memcache_ini = RawConfigParser()
memcache_ini.read(os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg'))
nodes = [node.strip() for node in memcache_ini.get('main', 'nodes').split(',')]
return [memcache_ini.get(node, 'location') for node in nodes]
def clean_logs():
"""
Cleans audit trail logs
"""
days = int(Configuration.get('ovs.core.audittrails.keep'))
mark = time.time() - days * 24 * 60 * 60
for log in LogList.get_logs():
if log.time < mark:
log.delete()
def _has_plugin(self):
"""
Checks whether this BackendType has a plugin installed
"""
try:
return True if Configuration.get('ovs.plugins.backend.' +
self.code) else False
except:
return False
def test_single_node(self):
base_port = Configuration.getInt('ovs.ports.arakoon')
cluster = 'one'
node = sorted(TestArakoonInstaller.nodes.keys())[0]
ArakoonInstaller.create_cluster(cluster, node, [])
contents = SSHClient.load(node).file_read(self._get_config_path(cluster))
expected = TestArakoonInstaller.expected_global.format(cluster, TestArakoonInstaller.nodes[node])
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
self.assertEqual(contents.strip(), expected.strip())
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._has_lock = False
try:
logger.debug('DS executing tick')
self._mutex.acquire(wait=10)
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = Configuration.get('ovs.core.uniqueid')
try:
lock = self._persistent.get('{0}_lock'.format(self._namespace))
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
logger.debug('DS there was no lock in tick')
else:
if lock['name'] == node_name:
# The current node holds the lock
logger.debug('DS keeps own lock')
self._has_lock = True
elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
logger.debug('DS last lock refresh is {0}s old'.format(
node_timestamp - lock['timestamp']))
logger.debug(
'DS stealing lock from {0}'.format(lock['name']))
self._load_schedule()
self._has_lock = True
else:
logger.debug('DS lock is not ours')
if self._has_lock is True:
lock = {'name': node_name,
'timestamp': node_timestamp}
logger.debug('DS refreshing lock')
self._persistent.set('{0}_lock'.format(self._namespace), lock)
finally:
self._mutex.release()
if self._has_lock is True:
logger.debug('DS executing tick workload')
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
logger.debug('DS executing tick workload - done')
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
def _get_memcache_nodes():
"""
Get the memcache nodes
"""
memcache_ini = ConfigObj(os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg'))
nodes = memcache_ini['main']['nodes']
if not isinstance(nodes, list):
nodes = nodes.split(',')
nodes = [node.strip() for node in nodes]
return [memcache_ini[node]['location'] for node in nodes]
def __init__(self, vpool_name):
self._vpool = vpool_name
self._config_specfile = os.path.join(
Configuration.get('ovs.core.cfgdir'), 'templates',
'volumedriverfs.json')
if not os.path.exists(
os.path.join(Configuration.get('ovs.core.cfgdir'),
'voldrv_vpools')):
os.makedirs(
os.path.join(Configuration.get('ovs.core.cfgdir'),
'voldrv_vpools'))
self._config_file = os.path.join(Configuration.get('ovs.core.cfgdir'),
'voldrv_vpools',
'{}.json'.format(vpool_name))
self._config_tmpfile = os.path.join(
Configuration.get('ovs.core.cfgdir'), 'voldrv_vpools',
'{}.json.tmp'.format(vpool_name))
self._config_readfile_handler = None
self._config_file_handler = None
self._config_specfile_handler = None
self._config_file_content = None
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._has_lock = False
try:
logger.debug("DS executing tick")
self._mutex.acquire(wait=10)
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = Configuration.get("ovs.core.uniqueid")
try:
lock = self._persistent.get("{0}_lock".format(self._namespace))
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
logger.debug("DS there was no lock in tick")
else:
if lock["name"] == node_name:
# The current node holds the lock
logger.debug("DS keeps own lock")
self._has_lock = True
elif node_timestamp - lock["timestamp"] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
logger.debug("DS last lock refresh is {0}s old".format(node_timestamp - lock["timestamp"]))
logger.debug("DS stealing lock from {0}".format(lock["name"]))
self._load_schedule()
self._has_lock = True
else:
logger.debug("DS lock is not ours")
if self._has_lock is True:
lock = {"name": node_name, "timestamp": node_timestamp}
logger.debug("DS refreshing lock")
self._persistent.set("{0}_lock".format(self._namespace), lock)
finally:
self._mutex.release()
if self._has_lock is True:
logger.debug("DS executing tick workload")
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
logger.debug("DS executing tick workload - done")
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
def run_event_consumer():
"""
Check whether to run the event consumer
"""
rmq_ini = ConfigObj(
os.path.join(Configuration.get('ovs.core.cfgdir'),
'rabbitmqclient.cfg'))
rmq_nodes = rmq_ini.get('main')['nodes'] if type(
rmq_ini.get('main')['nodes']) == list else [
rmq_ini.get('main')['nodes'],
]
machine_id = System.get_my_machine_id()
return machine_id in rmq_nodes
def get_client(client_type=None):
"""
Returns a persistent storage client
"""
if not hasattr(PersistentFactory,
'store') or PersistentFactory.store is None:
if client_type is None:
client_type = Configuration.get('ovs.core.storage.persistent')
PersistentFactory.store = None
if client_type == 'arakoon':
from ovs.extensions.storage.persistent.arakoonstore import ArakoonStore
cluster = Configuration.get('ovs.core.db.arakoon.clusterid')
PersistentFactory.store = ArakoonStore(cluster)
if client_type == 'default':
from ovs.extensions.storage.persistent.dummystore import DummyPersistentStore
PersistentFactory.store = DummyPersistentStore()
if PersistentFactory.store is None:
raise RuntimeError('Invalid client_type specified')
return PersistentFactory.store
def get_client(client_type=None):
"""
Returns a persistent storage client
"""
if not hasattr(PersistentFactory, "store") or PersistentFactory.store is None:
if client_type is None:
client_type = Configuration.get("ovs.core.storage.persistent")
PersistentFactory.store = None
if client_type == "arakoon":
from ovs.extensions.storage.persistent.arakoonstore import ArakoonStore
cluster = Configuration.get("ovs.core.db.arakoon.clusterid")
PersistentFactory.store = ArakoonStore(cluster)
if client_type == "default":
from ovs.extensions.storage.persistent.dummystore import DummyPersistentStore
PersistentFactory.store = DummyPersistentStore()
if PersistentFactory.store is None:
raise RuntimeError("Invalid client_type specified")
return PersistentFactory.store
def mds_checkup():
"""
Validates the current MDS setup/configuration and takes actions where required
"""
mds_dict = {}
for vpool in VPoolList.get_vpools():
for mds_service in vpool.mds_services:
storagerouter = mds_service.service.storagerouter
if vpool not in mds_dict:
mds_dict[vpool] = {}
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = []
mds_dict[vpool][storagerouter].append(mds_service)
for vpool in mds_dict:
# 1. First, make sure there's at least one MDS on every StorageRouter that's not overloaded
# If not, create an extra MDS for that StorageRouter
for storagerouter in mds_dict[vpool]:
mds_services = mds_dict[vpool][storagerouter]
has_room = False
for mds_service in mds_services:
load, _ = MDSServiceController.get_mds_load(mds_service)
if load < Configuration.getInt('ovs.storagedriver.mds.maxload'):
has_room = True
break
if has_room is False:
client = SSHClient.load(storagerouter.ip)
mds_service = MDSServiceController.prepare_mds_service(client, storagerouter, vpool,
fresh_only=False, start=True)
if mds_service is None:
raise RuntimeError('Could not add MDS node')
mds_dict[vpool][storagerouter].append(mds_service)
mds_config_set = MDSServiceController.get_mds_storagedriver_config_set(vpool)
for storagerouter in mds_dict[vpool]:
client = SSHClient.load(storagerouter.ip)
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
if storagedriver_config.is_new is False:
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_filesystem(
fs_metadata_backend_mds_nodes=mds_config_set[storagerouter.guid]
)
storagedriver_config.save(client)
# 2. Per VPool, execute a safety check, making sure the master/slave configuration is optimal.
for vdisk in vpool.vdisks:
MDSServiceController.ensure_safety(vdisk)
def get_my_machine_id(client=None):
"""
Returns unique machine id based on mac address
"""
if not System.my_machine_id:
ip_path = Configuration.get('ovs.core.ip.path')
if ip_path is None:
ip_path = "`which ip`"
cmd = """{0} a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g' | sort""".format(ip_path)
if client is None:
output = check_output(cmd, shell=True).strip()
else:
output = client.run(cmd).strip()
for mac in output.split('\n'):
if mac.strip() != '000000000000':
System.my_machine_id = mac.strip()
break
return System.my_machine_id
def collapse_arakoon():
logger.info('Starting arakoon collapse')
arakoon_dir = os.path.join(Configuration.get('ovs.core.cfgdir'), 'arakoon')
arakoon_clusters = map(lambda directory: os.path.basename(directory.rstrip(os.path.sep)),
os.walk(arakoon_dir).next()[1])
for cluster in arakoon_clusters:
logger.info(' Collapsing cluster: {}'.format(cluster))
cluster_instance = ArakoonManagementEx().getCluster(cluster)
for node in cluster_instance.listNodes():
logger.info(' Collapsing node: {}'.format(node))
try:
cluster_instance.remoteCollapse(node, 2) # Keep 2 tlogs
except Exception as e:
logger.info(
'Error during collapsing cluster {} node {}: {}\n{}'.format(
cluster, node, str(e), traceback.format_exc()
)
)
logger.info('Arakoon collapse finished')
def get_my_machine_id(client=None):
"""
Returns unique machine id based on mac address
"""
if not System.my_machine_id:
ip_path = Configuration.get('ovs.core.ip.path')
if ip_path is None:
ip_path = "`which ip`"
cmd = """{0} a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g' | sort""".format(
ip_path)
if client is None:
output = check_output(cmd, shell=True).strip()
else:
output = client.run(cmd).strip()
for mac in output.split('\n'):
if mac.strip() != '000000000000':
System.my_machine_id = mac.strip()
break
return System.my_machine_id
def test_multi_node(self):
base_port = Configuration.getInt('ovs.ports.arakoon')
cluster = 'one'
nodes = sorted(TestArakoonInstaller.nodes.keys())
ArakoonInstaller.create_cluster(cluster, nodes[0], [])
for node in nodes[1:]:
ArakoonInstaller.extend_cluster(nodes[0], node, cluster, [])
expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes))
for node in nodes:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
expected = expected.strip()
for node in nodes:
contents = SSHClient.load(node).file_read(self._get_config_path(cluster))
self.assertEqual(contents.strip(), expected.strip())
node = nodes[0]
ArakoonInstaller.shrink_cluster(nodes[1], node, cluster)
expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:]))
for node in nodes[1:]:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
expected = expected.strip()
for node in nodes[1:]:
contents = SSHClient.load(node).file_read(self._get_config_path(cluster))
self.assertEqual(contents.strip(), expected.strip())
def services_running(target):
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if target == 'framework':
# Volatile
_log(target, 'Testing volatile store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.storage.volatilefactory import VolatileFactory
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
except Exception as message:
_log(
target,
' Error during volatile store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
_log(target, ' Volatile store not working correctly', 2)
return False
_log(target, ' Volatile store OK after {0} tries'.format(tries),
0)
# Persistent
_log(target, 'Testing persistent store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.storage.persistentfactory import PersistentFactory
PersistentFactory.store = None
persistent = PersistentFactory.get_client()
persistent.set(key, value)
if persistent.get(key) == value:
persistent.delete(key)
break
persistent.delete(key)
except Exception as message:
_log(
target,
' Error during persistent store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
_log(target, ' Persistent store not working correctly', 2)
return False
_log(target, ' Persistent store OK after {0} tries'.format(tries),
0)
if target == 'volumedriver':
# Arakoon, voldrv cluster
_log(target, 'Testing arakoon (voldrv)...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagementEx
cluster = ArakoonManagementEx().getCluster('voldrv')
client = cluster.getClient()
client.set(key, value)
if client.get(key) == value:
client.delete(key)
break
client.delete(key)
except Exception as message:
_log(
target,
' Error during arakoon (voldrv) test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
_log(target, ' Arakoon (voldrv) not working correctly', 2)
return False
_log(target, ' Arakoon (voldrv) OK', 0)
if target in ['framework', 'volumedriver']:
# RabbitMQ
_log(target, 'Test rabbitMQ...', 0)
import pika
from configobj import ConfigObj
from ovs.plugin.provider.configuration import Configuration
rmq_ini = ConfigObj(
os.path.join(Configuration.get('ovs.core.cfgdir'),
'rabbitmqclient.cfg'))
rmq_nodes = rmq_ini.get('main')['nodes'] if type(
rmq_ini.get('main')['nodes']) == list else [
rmq_ini.get('main')['nodes']
]
rmq_servers = map(lambda m: rmq_ini.get(m)['location'], rmq_nodes)
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(
Configuration.get('ovs.core.broker.protocol'),
Configuration.get('ovs.core.broker.login'),
Configuration.get('ovs.core.broker.password'), server)
connection = pika.BlockingConnection(
pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish(
'', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain',
delivery_mode=1))
connection.close()
good_node = True
break
except Exception as message:
_log(
target,
' Error during rabbitMQ test on node {0}: {1}'.format(
server, message), 2)
if good_node is False:
_log(target, ' No working rabbitMQ node could be found', 2)
return False
_log(target, ' RabbitMQ test OK', 0)
_log(target, 'All tests OK', 1)
return True
except Exception as ex:
_log(target, 'Unexpected exception: {0}'.format(ex), 2)
return False
for task in return_data['tasks']:
self._process_task(task['task'], task['metadata'])
except Exception, ex:
logger.exception('Unexpected error processing tasks: {0}'.format(ex))
raise
if 'interval' in return_data:
interval = return_data['interval']
if interval != self.interval:
self.interval = interval
self._update_config('interval', str(interval))
self.interval = return_data['interval']
if __name__ == '__main__':
try:
if int(Configuration.get('ovs.support.enabled')) == 0:
print 'Support not enabled'
sys.exit(0)
logger.info('Starting up')
client = SupportAgent()
while True:
try:
client.run()
time.sleep(client.interval)
except KeyboardInterrupt:
raise
except Exception, exception:
logger.exception('Unexpected error during run: {0}'.format(exception))
time.sleep(10)
except KeyboardInterrupt:
print 'Aborting...'
and 'object' in [base.__name__ for base in member[1].__bases__]:
this_mapping = member[1].mapping
for key in this_mapping.keys():
if key not in mapping:
mapping[key] = []
mapping[key] += this_mapping[key]
logger.debug('Event map:')
for key in mapping:
logger.debug('{0}: {1}'.format(mapping[key][0]['property'], [
current_map['task'].__name__
for current_map in mapping[key]
]))
# Starting connection and handling
rmq_ini = ConfigObj(
os.path.join(Configuration.get('ovs.core.cfgdir'),
'rabbitmqclient.cfg'))
rmq_nodes = rmq_ini.get('main')['nodes'] if type(
rmq_ini.get('main')['nodes']) == list else [
rmq_ini.get('main')['nodes']
]
this_server = '{0}:{1}'.format(
Configuration.get('ovs.grid.ip'),
Configuration.get('ovs.core.broker.port'))
rmq_servers = [this_server] + [
server for server in map(lambda m: rmq_ini.get(m)['location'],
rmq_nodes) if server != this_server
]
channel = None
server = ''
loglevel = logging.root.manager.disable # Workaround for disabling logging
def __init__(self):
self._config_corefile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-core.conf')
self._config_exportfile = os.path.join(Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-export.conf')
def _get_int(key):
return int(Configuration.get(key))
def ensure_safety(vdisk, excluded_storagerouters=None):
"""
Ensures (or tries to ensure) the safety of a given vdisk (except hypervisor).
Assumptions:
* A local overloaded master is better than a non-local non-overloaded master
* Prefer master/services to be on different hosts, a subsequent slave on the same node doesn't add safety
* Don't actively overload services (e.g. configure an MDS as slave causing it to get overloaded)
* Too much safety is not wanted (it adds loads to nodes while not required)
"""
logger.debug('Ensuring MDS safety for vdisk {0}'.format(vdisk.guid))
vdisk.reload_client()
if excluded_storagerouters is None:
excluded_storagerouters = []
maxload = Configuration.getInt('ovs.storagedriver.mds.maxload')
safety = Configuration.getInt('ovs.storagedriver.mds.safety')
tlogs = Configuration.getInt('ovs.storagedriver.mds.tlogs')
services = [mds_service.service for mds_service in vdisk.vpool.mds_services
if mds_service.service.storagerouter not in excluded_storagerouters]
nodes = set(service.storagerouter.ip for service in services)
services_load = {}
service_per_key = {}
for service in services:
load, load_plus = MDSServiceController.get_mds_load(service.mds_service)
services_load[service.guid] = load, load_plus
service_per_key['{0}:{1}'.format(service.storagerouter.ip, service.ports[0])] = service
# List current configuration and filter out excluded services
reconfigure_required = False
reconfigure_reasons = []
vdisk.invalidate_dynamics(['info', 'storagedriver_id', 'storagerouter_guid'])
configs = vdisk.info['metadata_backend_config']
for config in configs:
config['key'] = '{0}:{1}'.format(config['ip'], config['port'])
master_service = None
if len(configs) > 0:
config = configs[0]
if config['key'] in service_per_key:
master_service = service_per_key.get(config['key'])
configs.remove(config)
else:
reconfigure_required = True
reconfigure_reasons.append('Master ({0}:{1}) cannot be used anymore'.format(config['ip'], config['port']))
slave_services = []
for config in configs:
if config['key'] in service_per_key:
slave_services.append(service_per_key[config['key']])
else:
reconfigure_required = True
reconfigure_reasons.append('Slave ({0}:{1}) cannot be used anymore'.format(config['ip'], config['port']))
# Fix services_load
services_per_load = {}
for service in services:
if service == master_service or service in slave_services:
load = services_load[service.guid][0]
else:
load = services_load[service.guid][1]
services_load[service.guid] = load
if load not in services_per_load:
services_per_load[load] = []
services_per_load[load].append(service)
# Further checks if a reconfiguration is required.
service_nodes = []
if master_service is not None:
service_nodes.append(master_service.storagerouter.ip)
for service in slave_services:
ip = service.storagerouter.ip
if ip in service_nodes:
reconfigure_required = True
reconfigure_reasons.append('Multiple MDS services on the same node')
else:
service_nodes.append(ip)
if len(service_nodes) > safety:
# Too much safety
reconfigure_required = True
reconfigure_reasons.append('Too much safety')
if len(service_nodes) < safety and len(service_nodes) < len(nodes):
# Insufficient MDS services configured while there should be sufficient nodes available
reconfigure_required = True
reconfigure_reasons.append('Not enough safety')
if master_service is not None and services_load[master_service.guid] > maxload:
# The master service is overloaded
reconfigure_required = True
reconfigure_reasons.append('Master overloaded')
if master_service is not None and master_service.storagerouter_guid != vdisk.storagerouter_guid:
# The master is not local
reconfigure_required = True
reconfigure_reasons.append('Master is not local')
if any(service for service in slave_services if services_load[service.guid] > maxload):
# There's a slave service overloaded
reconfigure_required = True
reconfigure_reasons.append('One or more slaves overloaded')
if reconfigure_required is False:
logger.debug('No reconfiguration required for vdisk {0}'.format(vdisk.guid))
MDSServiceController.sync_vdisk_to_reality(vdisk)
return
logger.debug('Reconfiguration required for vdisk {0}:'.format(vdisk.guid))
for reason in reconfigure_reasons:
logger.debug('Reason: {0} - vdisk {1}'.format(reason, vdisk.guid))
# Prepare fresh configuration
new_services = []
# Check whether the master (if available) is non-local to the vdisk and/or is overloaded
master_ok = master_service is not None
if master_ok is True:
master_ok = master_service.storagerouter_guid == vdisk.storagerouter_guid and services_load[master_service.guid] <= maxload
if master_ok:
# Add this master to the fresh configuration
new_services.append(master_service)
else:
# Try to find the best non-overloaded local MDS (slave)
candidate_master = None
candidate_master_load = 0
local_mds = None
local_mds_load = 0
for service in services:
load = services_load[service.guid]
if load <= maxload and service.storagerouter_guid == vdisk.storagerouter_guid:
if local_mds is None or local_mds_load > load:
# This service is a non-overloaded local MDS
local_mds = service
local_mds_load = load
if service in slave_services:
if candidate_master is None or candidate_master_load > load:
# This service is a non-overloaded local slave
candidate_master = service
candidate_master_load = load
if candidate_master is not None:
# A non-overloaded local slave was found.
client = MetadataServerClient.load(candidate_master)
amount_of_tlogs = client.catch_up(str(vdisk.volume_id), True)
if amount_of_tlogs < tlogs:
# Almost there. Catching up right now, and continue as soon as it's up-to-date
start = time.time()
client.catch_up(str(vdisk.volume_id), False)
logger.debug('MDS catch up for vdisk {0} took {1}s'.format(vdisk.guid, round(time.time() - start, 2)))
# It's up to date, so add it as a new master
new_services.append(candidate_master)
if master_service is not None:
# The current master (if available) is now candidate for become one of the slaves
slave_services.append(master_service)
else:
# It's not up to date, keep the previous master (if available) and give the local slave
# some more time to catch up
if master_service is not None:
new_services.append(master_service)
new_services.append(candidate_master)
if candidate_master in slave_services:
slave_services.remove(candidate_master)
else:
# There's no non-overloaded local slave found. Keep the current master (if available) and add
# a local MDS (if available) as slave
if master_service is not None:
new_services.append(master_service)
if local_mds is not None:
new_services.append(local_mds)
if local_mds in slave_services:
slave_services.remove(local_mds)
# At this point, there might (or might not) be a (new) master, and a (catching up) slave. The rest of the non-local
# MDS nodes must now be added to the configuration until the safety is reached. There's always one extra
# slave recycled to make sure there's always an (almost) up-to-date slave ready for failover
loads = sorted(load for load in services_per_load.keys() if load <= maxload)
nodes = set(service.storagerouter.ip for service in new_services)
slave_added = False
if len(nodes) < safety:
for load in loads:
for service in services_per_load[load]:
if slave_added is False and service in slave_services and service.storagerouter.ip not in nodes:
new_services.append(service)
slave_services.remove(service)
nodes.add(service.storagerouter.ip)
slave_added = True
if len(nodes) < safety:
for load in loads:
for service in services_per_load[load]:
if len(nodes) < safety and service.storagerouter.ip not in nodes:
new_services.append(service)
nodes.add(service.storagerouter.ip)
# Build the new configuration and update the vdisk
configs = []
for service in new_services:
client = MetadataServerClient.load(service)
client.create_namespace(str(vdisk.volume_id))
configs.append(MDSNodeConfig(address=str(service.storagerouter.ip),
port=service.ports[0]))
vdisk.storagedriver_client.update_metadata_backend_config(
volume_id=str(vdisk.volume_id),
metadata_backend_config=MDSMetaDataBackendConfig(configs)
)
MDSServiceController.sync_vdisk_to_reality(vdisk)
logger.debug('Ensuring MDS safety for vdisk {0} completed'.format(vdisk.guid))
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from subprocess import check_output
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.extensions.generic.system import System
from ovs.plugin.provider.configuration import Configuration
current_time = int(time.time())
machine_id = System.get_my_machine_id()
amqp = '{0}://{1}:{2}@{3}//'.format(Configuration.get('ovs.core.broker.protocol'),
Configuration.get('ovs.core.broker.login'),
Configuration.get('ovs.core.broker.password'),
Configuration.get('ovs.grid.ip'))
celery_path = Configuration.get('ovs.core.celery.path')
worker_states = check_output("{0} inspect ping -b {1} 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true".format(celery_path, amqp), shell=True)
routers = StorageRouterList.get_storagerouters()
for node in routers:
if node.heartbeats is None:
node.heartbeats = {}
if 'celery@{0}: OK'.format(node.name) in worker_states:
node.heartbeats['celery'] = current_time
if node.machine_id == machine_id:
node.heartbeats['process'] = current_time
node.save()
def add_vpool(parameters):
"""
Add a vPool to the machine this task is running on
"""
parameters = {} if parameters is None else parameters
ip = parameters['storagerouter_ip']
vpool_name = parameters['vpool_name']
if StorageRouterController._validate_ip(ip) is False:
raise ValueError('The entered ip address is invalid')
if not re.match('^[0-9a-z]+(\-+[0-9a-z]+)*$', vpool_name):
raise ValueError('Invalid vpool_name given')
client = SSHClient.load(ip) # Make sure to ALWAYS reload the client, as Fabric seems to be singleton-ish
unique_id = System.get_my_machine_id(client)
storagerouter = None
for current_storagerouter in StorageRouterList.get_storagerouters():
if current_storagerouter.ip == ip and current_storagerouter.machine_id == unique_id:
storagerouter = current_storagerouter
break
if storagerouter is None:
raise RuntimeError('Could not find Storage Router with given ip address')
vpool = VPoolList.get_vpool_by_name(vpool_name)
storagedriver = None
if vpool is not None:
if vpool.backend_type.code == 'local':
# Might be an issue, investigating whether it's on the same not or not
if len(vpool.storagedrivers) == 1 and vpool.storagedrivers[0].storagerouter.machine_id != unique_id:
raise RuntimeError('A local vPool with name {0} already exists'.format(vpool_name))
for vpool_storagedriver in vpool.storagedrivers:
if vpool_storagedriver.storagerouter_guid == storagerouter.guid:
storagedriver = vpool_storagedriver # The vPool is already added to this Storage Router and this might be a cleanup/recovery
# Check whether there are running machines on this vPool
machine_guids = []
for vdisk in vpool.vdisks:
if vdisk.vmachine_guid not in machine_guids:
machine_guids.append(vdisk.vmachine_guid)
if vdisk.vmachine.hypervisor_status in ['RUNNING', 'PAUSED']:
raise RuntimeError(
'At least one vMachine using this vPool is still running or paused. Make sure there are no active vMachines'
)
nodes = {ip} # Set comprehension
if vpool is not None:
for vpool_storagedriver in vpool.storagedrivers:
nodes.add(vpool_storagedriver.storagerouter.ip)
nodes = list(nodes)
services = ['volumedriver_{0}'.format(vpool_name),
'failovercache_{0}'.format(vpool_name)]
# Stop services
for node in nodes:
node_client = SSHClient.load(node)
for service in services:
System.exec_remote_python(node_client, """
from ovs.plugin.provider.service import Service
if Service.has_service('{0}'):
Service.disable_service('{0}')
""".format(service))
System.exec_remote_python(node_client, """
from ovs.plugin.provider.service import Service
if Service.has_service('{0}'):
Service.stop_service('{0}')
""".format(service))
# Keep in mind that if the Storage Driver exists, the vPool does as well
client = SSHClient.load(ip)
mountpoint_bfs = ''
directories_to_create = []
if vpool is None:
vpool = VPool()
supported_backends = System.read_remote_config(client, 'volumedriver.supported.backends').split(',')
if 'rest' in supported_backends:
supported_backends.remove('rest') # REST is not supported for now
backend_type = BackendTypeList.get_backend_type_by_code(parameters['type'])
vpool.backend_type = backend_type
connection_host = connection_port = connection_username = connection_password = None
if vpool.backend_type.code in ['local', 'distributed']:
vpool.metadata = {'backend_type': 'LOCAL'}
mountpoint_bfs = parameters['mountpoint_bfs']
directories_to_create.append(mountpoint_bfs)
vpool.metadata['local_connection_path'] = mountpoint_bfs
if vpool.backend_type.code == 'rest':
connection_host = parameters['connection_host']
connection_port = parameters['connection_port']
rest_connection_timeout_secs = parameters['connection_timeout']
vpool.metadata = {'rest_connection_host': connection_host,
'rest_connection_port': connection_port,
'buchla_connection_log_level': "0",
'rest_connection_verbose_logging': rest_connection_timeout_secs,
'rest_connection_metadata_format': "JSON",
'backend_type': 'REST'}
elif vpool.backend_type.code in ('ceph_s3', 'amazon_s3', 'swift_s3'):
connection_host = parameters['connection_host']
connection_port = parameters['connection_port']
connection_username = parameters['connection_username']
connection_password = parameters['connection_password']
if vpool.backend_type.code in ['swift_s3']:
strict_consistency = 'false'
s3_connection_flavour = 'SWIFT'
else:
strict_consistency = 'true'
s3_connection_flavour = 'S3'
vpool.metadata = {'s3_connection_host': connection_host,
's3_connection_port': connection_port,
's3_connection_username': connection_username,
's3_connection_password': connection_password,
's3_connection_flavour': s3_connection_flavour,
's3_connection_strict_consistency': strict_consistency,
's3_connection_verbose_logging': 1,
'backend_type': 'S3'}
vpool.name = vpool_name
vpool.description = "{} {}".format(vpool.backend_type.code, vpool_name)
vpool.login = connection_username
vpool.password = connection_password
if not connection_host:
vpool.connection = None
else:
vpool.connection = '{}:{}'.format(connection_host, connection_port)
vpool.save()
# Connection information is Storage Driver related information
new_storagedriver = False
if storagedriver is None:
storagedriver = StorageDriver()
new_storagedriver = True
mountpoint_temp = parameters['mountpoint_temp']
mountpoint_md = parameters['mountpoint_md']
mountpoint_readcache1 = parameters['mountpoint_readcache1']
mountpoint_readcache2 = parameters.get('mountpoint_readcache2', '')
mountpoint_writecache = parameters['mountpoint_writecache']
mountpoint_foc = parameters['mountpoint_foc']
directories_to_create.append(mountpoint_temp)
directories_to_create.append(mountpoint_md)
directories_to_create.append(mountpoint_readcache1)
if mountpoint_readcache2:
directories_to_create.append(mountpoint_readcache2)
directories_to_create.append(mountpoint_writecache)
directories_to_create.append(mountpoint_foc)
client = SSHClient.load(ip)
dir_create_script = """
import os
for directory in {0}:
if not os.path.exists(directory):
os.makedirs(directory)
""".format(directories_to_create)
System.exec_remote_python(client, dir_create_script)
read_cache1_fs = os.statvfs(mountpoint_readcache1)
read_cache2_fs = None
if mountpoint_readcache2:
read_cache2_fs = os.statvfs(mountpoint_readcache2)
write_cache_fs = os.statvfs(mountpoint_writecache)
fdcache = '{}/fd_{}'.format(mountpoint_writecache, vpool_name)
scocache = '{}/sco_{}'.format(mountpoint_writecache, vpool_name)
readcache1 = '{}/read1_{}'.format(mountpoint_readcache1, vpool_name)
files2create = [readcache1]
if mountpoint_readcache2 and mountpoint_readcache1 != mountpoint_readcache2:
readcache2 = '{}/read2_{}'.format(mountpoint_readcache2, vpool_name)
files2create.append(readcache2)
else:
readcache2 = ''
failovercache = '{}/foc_{}'.format(mountpoint_foc, vpool_name)
metadatapath = '{}/metadata_{}'.format(mountpoint_md, vpool_name)
tlogpath = '{}/tlogs_{}'.format(mountpoint_md, vpool_name)
rsppath = '/var/rsp/{}'.format(vpool_name)
dirs2create = [scocache, failovercache, metadatapath, tlogpath, rsppath,
System.read_remote_config(client, 'volumedriver.readcache.serialization.path')]
cmd = "cat /etc/mtab | grep ^/dev/ | cut -d ' ' -f 2"
mountpoints = [device.strip() for device in client.run(cmd).strip().split('\n')]
mountpoints.remove('/')
def is_partition(directory):
for mountpoint in mountpoints:
if directory == mountpoint:
return True
return False
# Cache sizes
# 20% = scocache
# 20% = failovercache (@TODO: check if this can possibly consume more than 20%)
# 60% = readcache
# safety values:
readcache1_factor = 0.2
readcache2_factor = 0.2
writecache_factor = 0.1
if (mountpoint_readcache1 == mountpoint_readcache2) or not mountpoint_readcache2:
delta = set()
delta.add(mountpoint_readcache1 if is_partition(mountpoint_readcache1) else '/dummy')
delta.add(mountpoint_writecache if is_partition(mountpoint_writecache) else '/dummy')
delta.add(mountpoint_foc if is_partition(mountpoint_foc) else '/dummy')
if len(delta) == 1:
readcache1_factor = 0.49
writecache_factor = 0.2
elif len(delta) == 2:
if mountpoint_writecache == mountpoint_foc:
readcache1_factor = 0.98
writecache_factor = 0.49
else:
readcache1_factor = 0.49
if mountpoint_readcache1 == mountpoint_writecache:
writecache_factor = 0.49
else:
writecache_factor = 0.98
elif len(delta) == 3:
readcache1_factor = 0.98
writecache_factor = 0.98
else:
delta = set()
delta.add(mountpoint_readcache1 if is_partition(mountpoint_readcache1) else '/dummy')
delta.add(mountpoint_readcache2 if is_partition(mountpoint_readcache2) else '/dummy')
delta.add(mountpoint_writecache if is_partition(mountpoint_writecache) else '/dummy')
delta.add(mountpoint_foc if is_partition(mountpoint_foc) else '/dummy')
if len(delta) == 1:
# consider them all to be directories
readcache1_factor = 0.24
readcache2_factor = 0.24
writecache_factor = 0.24
elif len(delta) == 2:
if mountpoint_writecache == mountpoint_foc:
writecache_factor = 0.24
if mountpoint_readcache1 == mountpoint_writecache:
readcache1_factor = 0.49
readcache2_factor = 0.98
else:
readcache1_factor = 0.98
readcache2_factor = 0.49
else:
readcache1_factor = readcache2_factor = 0.49
writecache_factor = 0.49
elif len(delta) == 3:
if mountpoint_writecache == mountpoint_foc:
readcache1_factor = 0.98
readcache2_factor = 0.98
writecache_factor = 0.49
elif mountpoint_readcache1 == mountpoint_writecache:
readcache1_factor = 0.49
readcache2_factor = 0.98
writecache_factor = 0.49
elif mountpoint_readcache1 == mountpoint_foc:
readcache1_factor = 0.49
readcache2_factor = 0.98
writecache_factor = 0.98
elif mountpoint_readcache2 == mountpoint_writecache:
readcache1_factor = 0.98
readcache2_factor = 0.49
writecache_factor = 0.49
elif mountpoint_readcache2 == mountpoint_foc:
readcache1_factor = 0.98
readcache2_factor = 0.49
writecache_factor = 0.98
elif len(delta) == 4:
readcache1_factor = 0.98
readcache2_factor = 0.98
writecache_factor = 0.98
# summarize caching on root partition (directory only)
root_assigned = dict()
if not is_partition(mountpoint_readcache1):
root_assigned['readcache1_factor'] = readcache1_factor
if not is_partition(mountpoint_readcache2):
root_assigned['readcache2_factor'] = readcache2_factor
if not is_partition(mountpoint_writecache):
root_assigned['writecache_factor'] = writecache_factor
if not is_partition(mountpoint_foc):
root_assigned['foc_factor'] = min(readcache1_factor, readcache2_factor, writecache_factor)
# always leave at least 20% of free space
division_factor = 1.0
total_size = sum(root_assigned.values()) + .02 * len(root_assigned)
if 0.8 < total_size < 1.6:
division_factor = 2.0
elif 1.6 < total_size < 3.2:
division_factor = 4.0
elif total_size >= 3.2:
division_factor = 8.0
if 'readcache1_factor' in root_assigned.keys():
readcache1_factor /= division_factor
if 'readcache2_factor' in root_assigned.keys():
readcache2_factor /= division_factor
if 'writecache_factor' in root_assigned.keys():
writecache_factor /= division_factor
scocache_size = '{0}KiB'.format((int(write_cache_fs.f_bavail * writecache_factor / 4096) * 4096) * 4)
if (mountpoint_readcache1 and not mountpoint_readcache2) or (mountpoint_readcache1 == mountpoint_readcache2):
mountpoint_readcache2 = ''
readcache1_size = '{0}KiB'.format((int(read_cache1_fs.f_bavail * readcache1_factor / 4096) * 4096) * 4)
readcache2 = ''
readcache2_size = '0KiB'
else:
readcache1_size = '{0}KiB'.format((int(read_cache1_fs.f_bavail * readcache1_factor / 4096) * 4096) * 4)
readcache2_size = '{0}KiB'.format((int(read_cache2_fs.f_bavail * readcache2_factor / 4096) * 4096) * 4)
if new_storagedriver:
ports_in_use = System.ports_in_use(client)
ports_reserved = []
ports_in_use_model = {}
for port_storagedriver in StorageDriverList.get_storagedrivers():
if port_storagedriver.vpool_guid not in ports_in_use_model:
ports_in_use_model[port_storagedriver.vpool_guid] = port_storagedriver.ports
ports_reserved += port_storagedriver.ports
if vpool.guid in ports_in_use_model: # The vPool is extended to another StorageRouter. We need to use these ports.
ports = ports_in_use_model[vpool.guid]
if any(port in ports_in_use for port in ports):
raise RuntimeError('The required ports are in use')
else: # First StorageDriver for this vPool, so generating new ports
ports = []
for port_range in System.read_remote_config(client, 'volumedriver.filesystem.ports').split(','):
port_range = port_range.strip()
if '-' in port_range:
current_range = (int(port_range.split('-')[0]), int(port_range.split('-')[1]))
else:
current_range = (int(port_range), 65536)
current_port = current_range[0]
while len(ports) < 3:
if current_port not in ports_in_use and current_port not in ports_reserved:
ports.append(current_port)
current_port += 1
if current_port > current_range[1]:
break
if len(ports) != 3:
raise RuntimeError('Could not find enough free ports')
else:
ports = storagedriver.ports
ip_path = Configuration.get('ovs.core.ip.path')
if ip_path is None:
ip_path = "`which ip`"
cmd = "{0} a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1".format(ip_path)
ipaddresses = client.run(cmd).strip().split('\n')
ipaddresses = [ipaddr.strip() for ipaddr in ipaddresses]
grid_ip = System.read_remote_config(client, 'ovs.grid.ip')
if grid_ip in ipaddresses:
ipaddresses.remove(grid_ip)
if not ipaddresses:
raise RuntimeError('No available ip addresses found suitable for Storage Router storage ip')
if storagerouter.pmachine.hvtype == 'KVM':
volumedriver_storageip = '127.0.0.1'
else:
volumedriver_storageip = parameters['storage_ip']
vrouter_id = '{0}{1}'.format(vpool_name, unique_id)
vrouter_config = {'vrouter_id': vrouter_id,
'vrouter_redirect_timeout_ms': '5000',
'vrouter_routing_retries': 10,
'vrouter_volume_read_threshold': 1024,
'vrouter_volume_write_threshold': 1024,
'vrouter_file_read_threshold': 1024,
'vrouter_file_write_threshold': 1024,
'vrouter_min_workers': 4,
'vrouter_max_workers': 16}
voldrv_arakoon_cluster_id = str(System.read_remote_config(client, 'volumedriver.arakoon.clusterid'))
voldrv_arakoon_cluster = ArakoonManagementEx().getCluster(voldrv_arakoon_cluster_id)
voldrv_arakoon_client_config = voldrv_arakoon_cluster.getClientConfig()
arakoon_node_configs = []
for arakoon_node in voldrv_arakoon_client_config.keys():
arakoon_node_configs.append(ArakoonNodeConfig(arakoon_node,
voldrv_arakoon_client_config[arakoon_node][0][0],
voldrv_arakoon_client_config[arakoon_node][1]))
vrouter_clusterregistry = ClusterRegistry(str(vpool.guid), voldrv_arakoon_cluster_id, arakoon_node_configs)
node_configs = []
for existing_storagedriver in StorageDriverList.get_storagedrivers():
if existing_storagedriver.vpool_guid == vpool.guid:
node_configs.append(ClusterNodeConfig(str(existing_storagedriver.storagedriver_id),
str(existing_storagedriver.cluster_ip),
existing_storagedriver.ports[0],
existing_storagedriver.ports[1],
existing_storagedriver.ports[2]))
if new_storagedriver:
node_configs.append(ClusterNodeConfig(vrouter_id, grid_ip, ports[0], ports[1], ports[2]))
vrouter_clusterregistry.set_node_configs(node_configs)
readcaches = [{'path': readcache1, 'size': readcache1_size}]
if readcache2:
readcaches.append({'path': readcache2, 'size': readcache2_size})
scocaches = [{'path': scocache, 'size': scocache_size}]
filesystem_config = {'fs_backend_path': mountpoint_bfs}
volumemanager_config = {'metadata_path': metadatapath, 'tlog_path': tlogpath}
storagedriver_config_script = """
from ovs.plugin.provider.configuration import Configuration
from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration
fd_config = {{'fd_cache_path': '{11}',
'fd_extent_cache_capacity': '1024',
'fd_namespace' : 'fd-{0}-{12}'}}
storagedriver_configuration = StorageDriverConfiguration('{0}')
storagedriver_configuration.configure_backend({1})
storagedriver_configuration.configure_readcache({2}, Configuration.get('volumedriver.readcache.serialization.path') + '/{0}')
storagedriver_configuration.configure_scocache({3}, '1GB', '2GB')
storagedriver_configuration.configure_failovercache('{4}')
storagedriver_configuration.configure_filesystem({5})
storagedriver_configuration.configure_volumemanager({6})
storagedriver_configuration.configure_volumerouter('{12}', {7})
storagedriver_configuration.configure_arakoon_cluster('{8}', {9})
storagedriver_configuration.configure_hypervisor('{10}')
storagedriver_configuration.configure_filedriver(fd_config)
""".format(vpool_name, vpool.metadata, readcaches, scocaches, failovercache, filesystem_config,
volumemanager_config, vrouter_config, voldrv_arakoon_cluster_id, voldrv_arakoon_client_config,
storagerouter.pmachine.hvtype, fdcache, vpool.guid)
System.exec_remote_python(client, storagedriver_config_script)
remote_script = """
import os
from configobj import ConfigObj
from ovs.plugin.provider.configuration import Configuration
protocol = Configuration.get('ovs.core.broker.protocol')
login = Configuration.get('ovs.core.broker.login')
password = Configuration.get('ovs.core.broker.password')
vpool_name = {0}
uris = []
cfg = ConfigObj('/opt/OpenvStorage/config/rabbitmqclient.cfg')
main_section = cfg.get('main')
nodes = main_section['nodes'] if type(main_section['nodes']) == list else [main_section['nodes']]
for node in nodes:
uris.append({{'amqp_uri': '{{0}}://{{1}}:{{2}}@{{3}}'.format(protocol, login, password, cfg.get(node)['location'])}})
from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration
queue_config = {{'events_amqp_routing_key': Configuration.get('ovs.core.broker.volumerouter.queue'),
'events_amqp_uris': uris}}
for config_file in os.listdir('/opt/OpenvStorage/config/voldrv_vpools'):
this_vpool_name = config_file.replace('.json', '')
if config_file.endswith('.json') and (vpool_name is None or vpool_name == this_vpool_name):
storagedriver_configuration = StorageDriverConfiguration(this_vpool_name)
storagedriver_configuration.configure_event_publisher(queue_config)
""".format(vpool_name if vpool_name is None else "'{0}'".format(vpool_name))
System.exec_remote_python(client, remote_script)
# Updating the model
storagedriver.storagedriver_id = vrouter_id
storagedriver.name = vrouter_id.replace('_', ' ')
storagedriver.description = storagedriver.name
storagedriver.storage_ip = volumedriver_storageip
storagedriver.cluster_ip = grid_ip
storagedriver.ports = ports
storagedriver.mountpoint = '/mnt/{0}'.format(vpool_name)
storagedriver.mountpoint_temp = mountpoint_temp
storagedriver.mountpoint_readcache1 = mountpoint_readcache1
storagedriver.mountpoint_readcache2 = mountpoint_readcache2
storagedriver.mountpoint_writecache = mountpoint_writecache
storagedriver.mountpoint_foc = mountpoint_foc
storagedriver.mountpoint_bfs = mountpoint_bfs
storagedriver.mountpoint_md = mountpoint_md
storagedriver.storagerouter = storagerouter
storagedriver.vpool = vpool
storagedriver.save()
dirs2create.append(storagedriver.mountpoint)
dirs2create.append(mountpoint_writecache + '/' + '/fd_' + vpool_name)
dirs2create.append('{0}/fd_{1}'.format(mountpoint_writecache, vpool_name))
file_create_script = """
import os
for directory in {0}:
if not os.path.exists(directory):
os.makedirs(directory)
for filename in {1}:
if not os.path.exists(filename):
open(filename, 'a').close()
""".format(dirs2create, files2create)
System.exec_remote_python(client, file_create_script)
voldrv_config_file = '{0}/voldrv_vpools/{1}.json'.format(System.read_remote_config(client, 'ovs.core.cfgdir'),
vpool_name)
log_file = '/var/log/ovs/volumedriver/{0}.log'.format(vpool_name)
vd_cmd = '/usr/bin/volumedriver_fs -f --config-file={0} --mountpoint {1} --logrotation --logfile {2} -o big_writes -o sync_read -o allow_other'.format(
voldrv_config_file, storagedriver.mountpoint, log_file)
if storagerouter.pmachine.hvtype == 'KVM':
vd_stopcmd = 'umount {0}'.format(storagedriver.mountpoint)
else:
vd_stopcmd = 'exportfs -u *:{0}; umount {0}'.format(storagedriver.mountpoint)
vd_name = 'volumedriver_{}'.format(vpool_name)
log_file = '/var/log/ovs/volumedriver/foc_{0}.log'.format(vpool_name)
fc_cmd = '/usr/bin/failovercachehelper --config-file={0} --logfile={1}'.format(voldrv_config_file, log_file)
fc_name = 'failovercache_{0}'.format(vpool_name)
params = {'<VPOOL_MOUNTPOINT>': storagedriver.mountpoint,
'<HYPERVISOR_TYPE>': storagerouter.pmachine.hvtype,
'<VPOOL_NAME>': vpool_name,
'<UUID>': str(uuid.uuid4())}
if Osdist.is_ubuntu(client):
if client.file_exists('/opt/OpenvStorage/config/templates/upstart/ovs-volumedriver.conf'):
client.run('cp -f /opt/OpenvStorage/config/templates/upstart/ovs-volumedriver.conf /opt/OpenvStorage/config/templates/upstart/ovs-volumedriver_{0}.conf'.format(vpool_name))
client.run('cp -f /opt/OpenvStorage/config/templates/upstart/ovs-failovercache.conf /opt/OpenvStorage/config/templates/upstart/ovs-failovercache_{0}.conf'.format(vpool_name))
else:
if client.file_exists('/opt/OpenvStorage/config/templates/systemd/ovs-volumedriver.service'):
client.run('cp -f /opt/OpenvStorage/config/templates/systemd/ovs-volumedriver.service /opt/OpenvStorage/config/templates/systemd/ovs-volumedriver_{0}.service'.format(vpool_name))
client.run('cp -f /opt/OpenvStorage/config/templates/systemd/ovs-failovercache.service /opt/OpenvStorage/config/templates/systemd/ovs-failovercache_{0}.service'.format(vpool_name))
service_script = """
from ovs.plugin.provider.service import Service
Service.add_service(package=('openvstorage', 'volumedriver'), name='{0}', command='{1}', stop_command='{2}', params={5})
Service.add_service(package=('openvstorage', 'failovercache'), name='{3}', command='{4}', stop_command=None, params={5})
""".format(
vd_name, vd_cmd, vd_stopcmd,
fc_name, fc_cmd, params
)
System.exec_remote_python(client, service_script)
if storagerouter.pmachine.hvtype == 'VMWARE':
client.run("grep -q '/tmp localhost(ro,no_subtree_check)' /etc/exports || echo '/tmp localhost(ro,no_subtree_check)' >> /etc/exports")
if Osdist.is_ubuntu(client):
client.run('service nfs-kernel-server start')
else:
client.run('service nfs start')
if storagerouter.pmachine.hvtype == 'KVM':
client.run('virsh pool-define-as {0} dir - - - - {1}'.format(vpool_name, storagedriver.mountpoint))
client.run('virsh pool-build {0}'.format(vpool_name))
client.run('virsh pool-start {0}'.format(vpool_name))
client.run('virsh pool-autostart {0}'.format(vpool_name))
# Start services
for node in nodes:
node_client = SSHClient.load(node)
for service in services:
System.exec_remote_python(node_client, """
from ovs.plugin.provider.service import Service
Service.enable_service('{0}')
""".format(service))
System.exec_remote_python(node_client, """
from ovs.plugin.provider.service import Service
Service.start_service('{0}')
""".format(service))
# Fill vPool size
vfs_info = os.statvfs('/mnt/{0}'.format(vpool_name))
vpool.size = vfs_info.f_blocks * vfs_info.f_bsize
vpool.save()
# Configure Cinder
ovsdb = PersistentFactory.get_client()
vpool_config_key = str('ovs_openstack_cinder_%s' % storagedriver.vpool_guid)
if ovsdb.exists(vpool_config_key):
# Second node gets values saved by first node
cinder_password, cinder_user, tenant_name, controller_ip, config_cinder = ovsdb.get(vpool_config_key)
else:
config_cinder = parameters.get('config_cinder', False)
cinder_password = ''
cinder_user = ''
tenant_name = ''
controller_ip = ''
if config_cinder:
cinder_password = parameters.get('cinder_pass', cinder_password)
cinder_user = parameters.get('cinder_user', cinder_user)
tenant_name = parameters.get('cinder_tenant', tenant_name)
controller_ip = parameters.get('cinder_controller', controller_ip) # Keystone host
if cinder_password:
osc = OpenStackCinder(cinder_password = cinder_password,
cinder_user = cinder_user,
tenant_name = tenant_name,
controller_ip = controller_ip)
osc.configure_vpool(vpool_name, storagedriver.mountpoint)
# Save values for first node to use
ovsdb.set(vpool_config_key,
[cinder_password, cinder_user, tenant_name, controller_ip, config_cinder])
def process(queue, body, mapping):
"""
Processes the actual received body
"""
if queue == Configuration.get('ovs.core.broker.volumerouter.queue'):
import json
cache = VolatileFactory.get_client()
all_extensions = None
message = FileSystemEvents.EventMessage()
message.ParseFromString(body)
# Possible special tags used as `arguments` key:
# - [NODE_ID]: Replaced by the storagedriver_id as reported by the event
# - [CLUSTER_ID]: Replaced by the clusterid as reported by the event
# Possible deduping key tags:
# - [EVENT_NAME]: The name of the eventmessage type
# - [TASK_NAME]: Task method name
# - [<argument value>]: Any value of the `arguments` dictionary.
logger.info('Got event, processing...')
event = None
for extension in mapping.keys():
if not message.event.HasExtension(extension):
continue
event = message.event.Extensions[extension]
node_id = message.node_id
cluster_id = message.cluster_id
for current_map in mapping[extension]:
task = current_map['task']
kwargs = {}
delay = 0
routing_key = 'generic'
for field, target in current_map['arguments'].iteritems():
if field == '[NODE_ID]':
kwargs[target] = node_id
elif field == '[CLUSTER_ID]':
kwargs[target] = cluster_id
else:
kwargs[target] = getattr(event, field)
if 'options' in current_map:
options = current_map['options']
if options.get('execonstoragerouter', False):
storagedriver = StorageDriverList.get_by_storagedriver_id(node_id)
if storagedriver is not None:
routing_key = 'sr.{0}'.format(storagedriver.storagerouter.machine_id)
delay = options.get('delay', 0)
dedupe = options.get('dedupe', False)
dedupe_key = options.get('dedupe_key', None)
if dedupe is True and dedupe_key is not None: # We can't dedupe without a key
key = dedupe_key
key = key.replace('[EVENT_NAME]', extension.full_name)
key = key.replace('[TASK_NAME]', task.__class__.__name__)
for kwarg_key in kwargs:
key = key.replace('[{0}]'.format(kwarg_key), kwargs[kwarg_key])
key = key.replace(' ', '_')
task_id = cache.get(key)
if task_id:
# Key exists, task was already scheduled
# If task is already running, the revoke message will
# be ignored
revoke(task_id)
_log(task, kwargs, node_id)
async_result = task.s(**kwargs).apply_async(
countdown=delay,
routing_key=routing_key
)
cache.set(key, async_result.id, 600) # Store the task id
new_task_id = async_result.id
else:
_log(task, kwargs, node_id)
async_result = task.s(**kwargs).apply_async(
countdown=delay,
routing_key=routing_key
)
new_task_id = async_result.id
else:
async_result = task.delay(**kwargs)
new_task_id = async_result.id
logger.info('[{0}] {1}({2}) started on {3} with taskid {4}. Delay: {5}s'.format(
queue,
task.__name__,
json.dumps(kwargs),
routing_key,
new_task_id,
delay
))
if event is None:
message_type = 'unknown'
if all_extensions is None:
all_extensions = _load_extensions()
for extension in all_extensions:
if message.event.HasExtension(extension):
message_type = extension.full_name
logger.info('A message with type {0} was received. Skipped.'.format(message_type))
else:
raise NotImplementedError('Queue {} is not yet implemented'.format(queue))
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Django settings module
"""
import os
from ovs.plugin.provider.configuration import Configuration
DEBUG = False
TEMPLATE_DEBUG = DEBUG
UNIQUE_ID = Configuration.get('ovs.core.uniqueid')
UI_NAME = Configuration.get('ovs.webapps.main.uiname')
APP_NAME = Configuration.get('ovs.webapps.main.appname')
BASE_WWW_DIR = os.path.dirname(__file__)
BASE_FOLDER = os.path.join(Configuration.get('ovs.core.basedir'), Configuration.get('ovs.webapps.dir'), APP_NAME)
VERSION = (1,) # This tuple should contain all supported versions. E.g.: (1,) or (1, 2) or (1, 2, 3) or (2, 3, 4) or ...
BASE_LOG_DIR = '/var/log/ovs'
LOG_FILENAME = '/django.log'
FRONTEND_ROOT = '/' + UI_NAME
STATIC_URL = '/' + UI_NAME + '/static/' # STATIC_URL must end with a slash
FORCE_SCRIPT_NAME = FRONTEND_ROOT
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Django settings module
"""
import os
from ovs.plugin.provider.configuration import Configuration
DEBUG = False
TEMPLATE_DEBUG = DEBUG
UI_NAME = Configuration.get('ovs.webapps.main.uiname')
APP_NAME = Configuration.get('ovs.webapps.main.appname')
BASE_WWW_DIR = os.path.dirname(__file__)
BASE_FOLDER = os.path.join(Configuration.get('ovs.core.basedir'),
Configuration.get('ovs.webapps.dir'), APP_NAME)
VERSION = (
1,
) # This tuple should contain all supported versions. E.g.: (1,) or (1, 2) or (1, 2, 3) or (2, 3, 4) or ...
BASE_LOG_DIR = '/var/log/ovs'
LOG_FILENAME = '/django.log'
FRONTEND_ROOT = '/' + UI_NAME
STATIC_URL = '/' + UI_NAME + '/static/' # STATIC_URL must end with a slash
def services_running(target):
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if target == 'framework':
# Volatile
_log(target, 'Testing volatile store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.storage.volatilefactory import VolatileFactory
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
except Exception as message:
_log(target, ' Error during volatile store test: {0}'.format(message), 2)
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
_log(target, ' Volatile store not working correctly', 2)
return False
_log(target, ' Volatile store OK after {0} tries'.format(tries), 0)
# Persistent
_log(target, 'Testing persistent store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.storage.persistentfactory import PersistentFactory
PersistentFactory.store = None
persistent = PersistentFactory.get_client()
persistent.set(key, value)
if persistent.get(key) == value:
persistent.delete(key)
break
persistent.delete(key)
except Exception as message:
_log(target, ' Error during persistent store test: {0}'.format(message), 2)
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
_log(target, ' Persistent store not working correctly', 2)
return False
_log(target, ' Persistent store OK after {0} tries'.format(tries), 0)
if target == 'volumedriver':
# Arakoon, voldrv cluster
_log(target, 'Testing arakoon (voldrv)...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagementEx
cluster = ArakoonManagementEx().getCluster('voldrv')
client = cluster.getClient()
client.set(key, value)
if client.get(key) == value:
client.delete(key)
break
client.delete(key)
except Exception as message:
_log(target, ' Error during arakoon (voldrv) test: {0}'.format(message), 2)
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
_log(target, ' Arakoon (voldrv) not working correctly', 2)
return False
_log(target, ' Arakoon (voldrv) OK', 0)
if target in ['framework', 'volumedriver']:
# RabbitMQ
_log(target, 'Test rabbitMQ...', 0)
import pika
from ConfigParser import RawConfigParser
from ovs.plugin.provider.configuration import Configuration
rmq_ini = RawConfigParser()
rmq_ini.read(os.path.join(Configuration.get('ovs.core.cfgdir'), 'rabbitmqclient.cfg'))
rmq_nodes = [node.strip() for node in rmq_ini.get('main', 'nodes').split(',')]
rmq_servers = map(lambda n: rmq_ini.get(n, 'location'), rmq_nodes)
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(Configuration.get('ovs.core.broker.protocol'),
Configuration.get('ovs.core.broker.login'),
Configuration.get('ovs.core.broker.password'),
server)
connection = pika.BlockingConnection(pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish('', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain', delivery_mode=1))
connection.close()
good_node = True
break
except Exception as message:
_log(target, ' Error during rabbitMQ test on node {0}: {1}'.format(server, message), 2)
if good_node is False:
_log(target, ' No working rabbitMQ node could be found', 2)
return False
_log(target, ' RabbitMQ test OK', 0)
_log(target, 'All tests OK', 1)
return True
except Exception as ex:
_log(target, 'Unexpected exception: {0}'.format(ex), 2)
return False
def process(queue, body, mapping):
"""
Processes the actual received body
"""
if queue == Configuration.get('ovs.core.broker.volumerouter.queue'):
import json
import volumedriver.storagerouter.EventMessages_pb2 as EventMessages
cache = VolatileFactory.get_client()
data = EventMessages.EventMessage().FromString(body)
# Possible special tags used as `arguments` key:
# - [NODE_ID]: Replaced by the storagedriver_id as reported by the event
# - [CLUSTER_ID]: Replaced by the clusterid as reported by the event
# Possible deduping key tags:
# - [EVENT_NAME]: The name of the eventmessage type
# - [TASK_NAME]: Task method name
# - [<argument value>]: Any value of the `arguments` dictionary.
if data.type in mapping:
for current_map in mapping[data.type]:
task = current_map['task']
data_container = getattr(data, current_map['property'])
kwargs = {}
delay = 0
routing_key = 'generic'
for field, target in current_map['arguments'].iteritems():
if field == '[NODE_ID]':
kwargs[target] = data.node_id
elif field == '[CLUSTER_ID]':
kwargs[target] = data.cluster_id
else:
kwargs[target] = getattr(data_container, field)
if 'options' in current_map:
options = current_map['options']
if options.get('execonstoragerouter', False):
storagedriver = StorageDriverList.get_by_storagedriver_id(
data.node_id)
if storagedriver is not None:
routing_key = 'sr.{0}'.format(
storagedriver.storagerouter.machine_id)
delay = options.get('delay', 0)
dedupe = options.get('dedupe', False)
dedupe_key = options.get('dedupe_key', None)
if dedupe is True and dedupe_key is not None: # We can't dedupe without a key
key = dedupe_key
key = key.replace('[EVENT_NAME]',
data.type.__class__.__name__)
key = key.replace('[TASK_NAME]',
task.__class__.__name__)
for kwarg_key in kwargs:
key = key.replace('[{0}]'.format(kwarg_key),
kwargs[kwarg_key])
key = key.replace(' ', '_')
task_id = cache.get(key)
if task_id:
# Key exists, task was already scheduled
# If task is already running, the revoke message will
# be ignored
revoke(task_id)
_log(task, kwargs, data.node_id)
async_result = task.s(**kwargs).apply_async(
countdown=delay, routing_key=routing_key)
cache.set(key, async_result.id,
600) # Store the task id
new_task_id = async_result.id
else:
_log(task, kwargs, data.node_id)
async_result = task.s(**kwargs).apply_async(
countdown=delay, routing_key=routing_key)
new_task_id = async_result.id
else:
async_result = task.delay(**kwargs)
new_task_id = async_result.id
logger.info(
'[{0}] {1}({2}) started on {3} with taskid {4}. Delay: {5}s'
.format(queue, task.__name__, json.dumps(kwargs),
routing_key, new_task_id, delay))
else:
logger.info('Message type {0} was received. Skipped.'.format(
str(data.type)))
else:
raise NotImplementedError(
'Queue {} is not yet implemented'.format(queue))
def prepare_mds_service(client, storagerouter, vpool, fresh_only=True, start=False):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
"""
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid][0]
# Fetch service sequence number
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return None # There are already one or more MDS services running, aborting
service_number += 1
# Find free port
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.append(service.ports[0])
port = System.get_free_ports(Configuration.get('ovs.ports.mds'),
exclude=occupied_ports, nr=1, client=client)[0]
# Add service to the model
service = Service()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name, service_number)
service.type = mdsservice_type
service.storagerouter = storagerouter
service.ports = [port]
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.save()
# Prepare some directores
scratch_dir = '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_temp, vpool.name, service_number)
rocksdb_dir = '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_md, vpool.name, service_number)
client.run('mkdir -p {0}'.format(scratch_dir))
client.run('mkdir -p {0}'.format(rocksdb_dir))
# Generate the configuration file
metadataserver_config = StorageDriverConfiguration('metadataserver', vpool.name, number=service_number)
metadataserver_config.load(client)
metadataserver_config.clean() # Clean out obsolete values
if vpool.backend_type.code == 'alba':
metadataserver_config.configure_backend_connection_manager(alba_connection_host='127.0.0.1',
alba_connection_port=storagedriver.alba_proxy.service.ports[0],
backend_type='ALBA')
else:
metadataserver_config.configure_backend_connection_manager(**vpool.metadata)
metadataserver_config.configure_metadata_server(mds_address=storagerouter.ip,
mds_port=service.ports[0],
mds_scratch_dir=scratch_dir,
mds_rocksdb_path=rocksdb_dir)
metadataserver_config.save(client)
# Create system services
params = {'<VPOOL_NAME>': vpool.name,
'<SERVICE_NUMBER>': str(service_number)}
template_dir = '/opt/OpenvStorage/config/templates/upstart'
client.run('cp -f {0}/ovs-metadataserver.conf {0}/ovs-metadataserver_{1}_{2}.conf'.format(template_dir, vpool.name, service_number))
service_script = """
from ovs.plugin.provider.service import Service
Service.add_service(package=('openvstorage', 'metadataserver'), name='metadataserver_{0}_{1}', command=None, stop_command=None, params={2})
""".format(vpool.name, service_number, params)
System.exec_remote_python(client, service_script)
if start is True:
System.exec_remote_python(client, """
from ovs.plugin.provider.service import Service
Service.enable_service('{0}')
""".format(service.name))
System.exec_remote_python(client, """
from ovs.plugin.provider.service import Service
Service.start_service('{0}')
""".format(service.name))
return mds_service
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from subprocess import check_output
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.extensions.generic.system import System
from ovs.plugin.provider.configuration import Configuration
current_time = int(time.time())
machine_id = System.get_my_machine_id()
amqp = '{0}://{1}:{2}@{3}//'.format(
Configuration.get('ovs.core.broker.protocol'),
Configuration.get('ovs.core.broker.login'),
Configuration.get('ovs.core.broker.password'),
Configuration.get('ovs.grid.ip'))
celery_path = Configuration.get('ovs.core.celery.path')
worker_states = check_output(
"{0} inspect ping -b {1} 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true"
.format(celery_path, amqp),
shell=True)
routers = StorageRouterList.get_storagerouters()
for node in routers:
if node.heartbeats is None:
node.heartbeats = {}
if 'celery@{0}: OK'.format(node.name) in worker_states:
node.heartbeats['celery'] = current_time
import os
from kombu import Queue
from celery import Celery
from celery.schedules import crontab
from celery.signals import task_postrun, worker_process_init
from ovs.lib.messaging import MessageController
from ovs.log.logHandler import LogHandler
from ovs.extensions.storage.volatilefactory import VolatileFactory
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.extensions.generic.system import System
from ovs.plugin.provider.configuration import Configuration
from configobj import ConfigObj
memcache_ini = ConfigObj(
os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg'))
memcache_nodes = memcache_ini.get('main')['nodes'] if type(
memcache_ini.get('main')['nodes']) == list else [
memcache_ini.get('main')['nodes'],
]
memcache_servers = map(lambda m: memcache_ini.get(m)['location'],
memcache_nodes)
rmq_ini = ConfigObj(
os.path.join(Configuration.get('ovs.core.cfgdir'), 'rabbitmqclient.cfg'))
rmq_nodes = rmq_ini.get('main')['nodes'] if type(
rmq_ini.get('main')['nodes']) == list else [
rmq_ini.get('main')['nodes'],
]
rmq_servers = map(lambda m: rmq_ini.get(m)['location'], rmq_nodes)
