def save(self, client=None, reload_config=True):
"""
Saves the configuration to a given file, optionally a remote one
:param client: If provided, save remote configuration
:param reload_config: Reload the running Storage Driver configuration
"""
self._validate()
contents = json.dumps(self.configuration, indent=4)
EtcdConfiguration.set(self.path, contents, raw=True)
if self.config_type == 'storagedriver' and reload_config is True:
if len(self.dirty_entries) > 0:
if client is None:
logger.info('Applying local storagedriver configuration changes')
changes = LSRClient(self.remote_path).update_configuration(self.remote_path)
else:
logger.info('Applying storagedriver configuration changes on {0}'.format(client.ip))
with Remote(client.ip, [LSRClient]) as remote:
changes = copy.deepcopy(remote.LocalStorageRouterClient(self.remote_path).update_configuration(self.remote_path))
for change in changes:
if change['param_name'] not in self.dirty_entries:
raise RuntimeError('Unexpected configuration change: {0}'.format(change['param_name']))
logger.info('Changed {0} from "{1}" to "{2}"'.format(change['param_name'], change['old_value'], change['new_value']))
self.dirty_entries.remove(change['param_name'])
logger.info('Changes applied')
if len(self.dirty_entries) > 0:
logger.warning('Following changes were not applied: {0}'.format(', '.join(self.dirty_entries)))
else:
logger.debug('No need to apply changes, nothing changed')
self.is_new = False
self.dirty_entries = []
def load_metadata(self):
"""
Reads the metadata for an arakoon cluster from reality
:return: None
"""
key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
if not EtcdConfiguration.exists(key):
return
metadata = EtcdConfiguration.get(key)
if not isinstance(metadata, dict):
raise ValueError('Metadata should be a dictionary')
for key in ['in_use', 'internal', 'type']:
if key not in metadata:
raise ValueError('Not all required metadata keys are present for arakoon cluster {0}'.format(self.cluster_id))
value = metadata[key]
if key == 'in_use':
if not isinstance(value, bool):
raise ValueError('"in_use" should be of type "bool"')
self.in_use = value
elif key == 'internal':
if not isinstance(value, bool):
raise ValueError('"internal" should be of type "bool"')
self.internal = value
else:
if value not in ServiceType.ARAKOON_CLUSTER_TYPES:
raise ValueError('Unsupported arakoon cluster type {0} found\nPlease choose from {1}'.format(value, ', '.join(ServiceType.ARAKOON_CLUSTER_TYPES)))
self.cluster_type = value
def migrate(master_ips=None, extra_ips=None):
"""
Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one
:param master_ips: IP addresses of the MASTER nodes
:param extra_ips: IP addresses of the EXTRA nodes
"""
data = EtcdConfiguration.get('/ovs/framework/versions') if EtcdConfiguration.exists('/ovs/framework/versions') else {}
migrators = []
path = os.path.join(os.path.dirname(__file__), 'migration')
for filename in os.listdir(path):
if os.path.isfile(os.path.join(path, filename)) and filename.endswith('.py'):
name = filename.replace('.py', '')
module = imp.load_source(name, os.path.join(path, filename))
for member in inspect.getmembers(module):
if inspect.isclass(member[1]) and member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]:
migrators.append((member[1].identifier, member[1].migrate))
end_version = 0
for identifier, method in migrators:
base_version = data[identifier] if identifier in data else 0
version = method(base_version, master_ips, extra_ips)
if version > end_version:
end_version = version
data[identifier] = end_version
EtcdConfiguration.set('/ovs/framework/versions', data)
def _get_free_ports(client):
node_name = System.get_my_machine_id(client)
clusters = []
exclude_ports = []
if EtcdConfiguration.dir_exists(ArakoonInstaller.ETCD_CONFIG_ROOT):
for cluster_name in EtcdConfiguration.list(
ArakoonInstaller.ETCD_CONFIG_ROOT):
try:
config = ArakoonClusterConfig(cluster_name)
config.load_config()
for node in config.nodes:
if node.name == node_name:
clusters.append(cluster_name)
exclude_ports.append(node.client_port)
exclude_ports.append(node.messaging_port)
except:
logger.error(
' Could not load port information of cluster {0}'.
format(cluster_name))
ports = System.get_free_ports(
EtcdConfiguration.get(
'/ovs/framework/hosts/{0}/ports|arakoon'.format(node_name)),
exclude_ports, 2, client)
logger.debug(
' Loaded free ports {0} based on existing clusters {1}'.format(
ports, clusters))
return ports
def get_unused_arakoon_metadata_and_claim(cluster_type, locked=True):
"""
Retrieve arakoon cluster information based on its type
:param cluster_type: Type of arakoon cluster (See ServiceType.ARAKOON_CLUSTER_TYPES)
:type cluster_type: str
:param locked: Execute this in a locked context
:type locked: bool
:return: List of ArakoonClusterMetadata objects
:rtype: ArakoonClusterMetadata
"""
cluster_type = cluster_type.upper()
if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES:
raise ValueError('Unsupported arakoon cluster type provided. Please choose from {0}'.format(', '.join(ServiceType.ARAKOON_CLUSTER_TYPES)))
if not EtcdConfiguration.dir_exists('/ovs/arakoon'):
return None
mutex = volatile_mutex('claim_arakoon_metadata', wait=10)
try:
if locked is True:
mutex.acquire()
for cluster_name in EtcdConfiguration.list('/ovs/arakoon'):
metadata = ArakoonClusterMetadata(cluster_id=cluster_name)
metadata.load_metadata()
if metadata.cluster_type == cluster_type and metadata.in_use is False and metadata.internal is False:
metadata.claim()
return metadata
finally:
if locked is True:
mutex.release()
def _configure_arakoon_to_volumedriver():
print 'Update existing vPools'
logger.info('Update existing vPools')
config = ArakoonClusterConfig('voldrv')
config.load_config()
arakoon_nodes = []
for node in config.nodes:
arakoon_nodes.append({
'host': node.ip,
'port': node.client_port,
'node_id': node.name
})
if EtcdConfiguration.dir_exists('/ovs/vpools'):
for vpool_guid in EtcdConfiguration.list('/ovs/vpools'):
for storagedriver_id in EtcdConfiguration.list(
'/ovs/vpools/{0}/hosts'.format(vpool_guid)):
storagedriver_config = StorageDriverConfiguration(
'storagedriver', vpool_guid, storagedriver_id)
storagedriver_config.load()
storagedriver_config.configure_volume_registry(
vregistry_arakoon_cluster_id='voldrv',
vregistry_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.configure_distributed_lock_store(
dls_type='Arakoon',
dls_arakoon_cluster_id='voldrv',
dls_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.save(reload_config=True)
def delete_config(self):
"""
Deletes a configuration file
"""
EtcdConfiguration.delete(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(
self.cluster_id),
raw=True)
def get_client(client_type=None):
"""
Returns a persistent storage client
:param client_type: Type of store client
"""
if not hasattr(PersistentFactory,
'store') or PersistentFactory.store is None:
if hasattr(unittest, 'running_tests') and getattr(
unittest, 'running_tests'):
client_type = 'dummy'
if client_type is None:
client_type = EtcdConfiguration.get(
'/ovs/framework/stores|persistent')
PersistentFactory.store = None
if client_type in ['pyrakoon', 'arakoon']:
from ovs.extensions.storage.persistent.pyrakoonstore import PyrakoonStore
PersistentFactory.store = PyrakoonStore(
str(
EtcdConfiguration.get(
'/ovs/framework/arakoon_clusters|ovsdb')))
if client_type == 'dummy':
from ovs.extensions.storage.persistent.dummystore import DummyPersistentStore
PersistentFactory.store = DummyPersistentStore()
if PersistentFactory.store is None:
raise RuntimeError('Invalid client_type specified')
return PersistentFactory.store
def migrate(master_ips=None, extra_ips=None):
"""
Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one
:param master_ips: IP addresses of the MASTER nodes
:param extra_ips: IP addresses of the EXTRA nodes
"""
data = EtcdConfiguration.get(
'/ovs/framework/versions') if EtcdConfiguration.exists(
'/ovs/framework/versions') else {}
migrators = []
path = os.path.join(os.path.dirname(__file__), 'migration')
for filename in os.listdir(path):
if os.path.isfile(os.path.join(
path, filename)) and filename.endswith('.py'):
name = filename.replace('.py', '')
module = imp.load_source(name, os.path.join(path, filename))
for member in inspect.getmembers(module):
if inspect.isclass(
member[1]
) and member[1].__module__ == name and 'object' in [
base.__name__ for base in member[1].__bases__
]:
migrators.append(
(member[1].identifier, member[1].migrate))
end_version = 0
for identifier, method in migrators:
base_version = data[identifier] if identifier in data else 0
version = method(base_version, master_ips, extra_ips)
if version > end_version:
end_version = version
data[identifier] = end_version
EtcdConfiguration.set('/ovs/framework/versions', data)
def get_unused_arakoon_metadata_and_claim(cluster_type, locked=True):
"""
Retrieve arakoon cluster information based on its type
:param cluster_type: Type of arakoon cluster (See ServiceType.ARAKOON_CLUSTER_TYPES)
:type cluster_type: str
:param locked: Execute this in a locked context
:type locked: bool
:return: List of ArakoonClusterMetadata objects
:rtype: ArakoonClusterMetadata
"""
cluster_type = cluster_type.upper()
if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES:
raise ValueError('Unsupported arakoon cluster type provided. Please choose from {0}'.format(', '.join(ServiceType.ARAKOON_CLUSTER_TYPES)))
if not EtcdConfiguration.dir_exists('/ovs/arakoon'):
return None
mutex = volatile_mutex('claim_arakoon_metadata', wait=10)
try:
if locked is True:
mutex.acquire()
for cluster_name in EtcdConfiguration.list('/ovs/arakoon'):
metadata = ArakoonClusterMetadata(cluster_id=cluster_name)
metadata.load_metadata()
if metadata.cluster_type == cluster_type and metadata.in_use is False and metadata.internal is False:
metadata.claim()
return metadata
finally:
if locked is True:
mutex.release()
def delete_config(self):
"""
Deletes a configuration file
"""
key = ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id)
if EtcdConfiguration.exists(key, raw=True):
EtcdConfiguration.delete(key, raw=True)
def load_metadata(self):
"""
Reads the metadata for an arakoon cluster from reality
:return: None
"""
key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
if not EtcdConfiguration.exists(key):
return
metadata = EtcdConfiguration.get(key)
if not isinstance(metadata, dict):
raise ValueError('Metadata should be a dictionary')
for key in ['in_use', 'internal', 'type']:
if key not in metadata:
raise ValueError('Not all required metadata keys are present for arakoon cluster {0}'.format(self.cluster_id))
value = metadata[key]
if key == 'in_use':
if not isinstance(value, bool):
raise ValueError('"in_use" should be of type "bool"')
self.in_use = value
elif key == 'internal':
if not isinstance(value, bool):
raise ValueError('"internal" should be of type "bool"')
self.internal = value
else:
if value not in ServiceType.ARAKOON_CLUSTER_TYPES:
raise ValueError('Unsupported arakoon cluster type {0} found\nPlease choose from {1}'.format(value, ', '.join(ServiceType.ARAKOON_CLUSTER_TYPES)))
self.cluster_type = value
def register(name, email, company, phone, newsletter):
"""
Registers the environment
"""
SupportAgent().run() # Execute a single heartbeat run
client = OVSClient('monitoring.openvstorage.com',
443,
credentials=None,
verify=True,
version=1)
task_id = client.post(
'/support/register/',
data={
'cluster_id':
EtcdConfiguration.get('/ovs/framework/cluster_id'),
'name': name,
'email': email,
'company': company,
'phone': phone,
'newsletter': newsletter,
'register_only': True
})
if task_id:
client.wait_for_task(task_id, timeout=120)
EtcdConfiguration.set('/ovs/framework/registered', True)
def pulse():
"""
Update the heartbeats for all Storage Routers
:return: None
"""
logger = LogHandler.get('extensions', name='heartbeat')
current_time = int(time.time())
machine_id = System.get_my_machine_id()
amqp = '{0}://{1}:{2}@{3}//'.format(EtcdConfiguration.get('/ovs/framework/messagequeue|protocol'),
EtcdConfiguration.get('/ovs/framework/messagequeue|user'),
EtcdConfiguration.get('/ovs/framework/messagequeue|password'),
EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(machine_id)))
celery_path = OSManager.get_path('celery')
worker_states = check_output("{0} inspect ping -b {1} --timeout=5 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true".format(celery_path, amqp), shell=True)
routers = StorageRouterList.get_storagerouters()
for node in routers:
if node.heartbeats is None:
node.heartbeats = {}
if 'celery@{0}: OK'.format(node.name) in worker_states:
node.heartbeats['celery'] = current_time
if node.machine_id == machine_id:
node.heartbeats['process'] = current_time
else:
try:
# check timeout of other nodes and clear arp cache
if node.heartbeats and 'process' in node.heartbeats:
if current_time - node.heartbeats['process'] >= HeartBeat.ARP_TIMEOUT:
check_output("/usr/sbin/arp -d {0}".format(node.name), shell=True)
except CalledProcessError:
logger.exception('Error clearing ARP cache')
node.save()
def teardown():
"""
Teardown for Arakoon package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is False:
GeneralService.start_service(name='ovs-scheduled-tasks',
client=root_client)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
for key in KEY_CLEANUP:
if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True):
EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))
def delete_config(self):
"""
Deletes a configuration file
"""
key = ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id)
if EtcdConfiguration.exists(key, raw=True):
EtcdConfiguration.delete(key, raw=True)
def run_event_consumer():
"""
Check whether to run the event consumer
"""
my_ip = EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(System.get_my_machine_id()))
for endpoint in EtcdConfiguration.get('/ovs/framework/messagequeue|endpoints'):
if endpoint.startswith(my_ip):
return True
return False
def delete_etcd_config(cluster_name):
"""
Remove the etcd entry for arakoon cluster_name
:param cluster_name: Name of the arakoon cluster
:return: None
"""
etcd_key = GeneralArakoon.ETCD_CONFIG_KEY.format(cluster_name)
if EtcdConfiguration.exists(etcd_key, raw=True):
EtcdConfiguration.delete(os.path.dirname(etcd_key))
def _process_disk(_info, _disks, _node):
disk = _info.get('disk')
if disk is None:
return
disk_status = 'uninitialized'
disk_status_detail = ''
disk_alba_backend_guid = ''
if disk['available'] is False:
osd = _info.get('osd')
disk_alba_state = disk['state']['state']
if disk_alba_state == 'ok':
if osd is None:
disk_status = 'initialized'
elif osd['id'] is None:
alba_id = osd['alba_id']
if alba_id is None:
disk_status = 'available'
else:
disk_status = 'unavailable'
alba_backend = alba_backend_map.get(alba_id)
if alba_backend is not None:
disk_alba_backend_guid = alba_backend.guid
else:
disk_status = 'error'
disk_status_detail = 'communicationerror'
disk_alba_backend_guid = self.guid
for asd in _node.asds:
if asd.asd_id == disk['asd_id'] and asd.statistics != {}:
disk_status = 'warning'
disk_status_detail = 'recenterrors'
read = osd['read'] or [0]
write = osd['write'] or [0]
errors = osd['errors']
global_interval_key = '/ovs/alba/backends/global_gui_error_interval'
backend_interval_key = '/ovs/alba/backends/{0}/gui_error_interval'.format(self.guid)
interval = EtcdConfiguration.get(global_interval_key)
if EtcdConfiguration.exists(backend_interval_key):
interval = EtcdConfiguration.get(backend_interval_key)
if len(errors) == 0 or (len(read + write) > 0 and max(min(read), min(write)) > max(error[0] for error in errors) + interval):
disk_status = 'claimed'
disk_status_detail = ''
elif disk_alba_state == 'decommissioned':
disk_status = 'unavailable'
disk_status_detail = 'decommissioned'
else:
disk_status = 'error'
disk_status_detail = disk['state']['detail']
alba_backend = alba_backend_map.get(osd.get('alba_id'))
if alba_backend is not None:
disk_alba_backend_guid = alba_backend.guid
disk['status'] = disk_status
disk['status_detail'] = disk_status_detail
disk['alba_backend_guid'] = disk_alba_backend_guid
_disks.append(disk)
def get_path(binary_name):
machine_id = System.get_my_machine_id()
config_location = '/ovs/framework/hosts/{0}/paths|{1}'.format(machine_id, binary_name)
path = EtcdConfiguration.get(config_location)
if not path:
try:
path = check_output('which {0}'.format(binary_name), shell=True).strip()
EtcdConfiguration.set(config_location, path)
except CalledProcessError:
return None
return path
def list(self, discover=False, ip=None, node_id=None):
"""
Lists all available ALBA Nodes
:param discover: If True and IP provided, return list of single ALBA node, If True and no IP provided, return all ALBA nodes else return modeled ALBA nodes
:param ip: IP of ALBA node to retrieve
:param node_id: ID of the ALBA node
"""
if discover is False and (ip is not None or node_id is not None):
raise RuntimeError('Discover is mutually exclusive with IP and nodeID')
if (ip is None and node_id is not None) or (ip is not None and node_id is None):
raise RuntimeError('Both IP and nodeID need to be specified')
if discover is False:
return AlbaNodeList.get_albanodes()
if ip is not None:
node = AlbaNode(volatile=True)
node.ip = ip
node.type = 'ASD'
node.node_id = node_id
node.port = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id))
node.username = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id))
node.password = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id))
data = node.client.get_metadata()
if data['_success'] is False and data['_error'] == 'Invalid credentials':
raise RuntimeError('Invalid credentials')
if data['node_id'] != node_id:
raise RuntimeError('Unexpected node identifier. {0} vs {1}'.format(data['node_id'], node_id))
node_list = DataList(AlbaNode, {})
node_list._executed = True
node_list._guids = [node.guid]
node_list._objects = {node.guid: node}
node_list._data = {node.guid: {'guid': node.guid, 'data': node._data}}
return node_list
nodes = {}
model_node_ids = [node.node_id for node in AlbaNodeList.get_albanodes()]
found_node_ids = []
asd_node_ids = []
if EtcdConfiguration.dir_exists('/ovs/alba/asdnodes'):
asd_node_ids = EtcdConfiguration.list('/ovs/alba/asdnodes')
for node_id in asd_node_ids:
node = AlbaNode(volatile=True)
node.type = 'ASD'
node.node_id = node_id
node.ip = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|ip'.format(node_id))
node.port = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id))
node.username = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id))
node.password = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id))
if node.node_id not in model_node_ids and node.node_id not in found_node_ids:
nodes[node.guid] = node
found_node_ids.append(node.node_id)
node_list = DataList(AlbaNode, {})
node_list._executed = True
node_list._guids = nodes.keys()
node_list._objects = nodes
node_list._data = dict([(node.guid, {'guid': node.guid, 'data': node._data}) for node in nodes.values()])
return node_list
def run_event_consumer():
"""
Check whether to run the event consumer
"""
my_ip = EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(
System.get_my_machine_id()))
for endpoint in EtcdConfiguration.get(
'/ovs/framework/messagequeue|endpoints'):
if endpoint.startswith(my_ip):
return True
return False
def write(self):
"""
Write the metadata to Etcd
:return: None
"""
if self.cluster_type is None or self.cluster_type == '':
raise ValueError('Cluster type must be defined before being able to store the cluster metadata information')
etcd_key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
EtcdConfiguration.set(key=etcd_key, value={'type': self.cluster_type,
'in_use': self.in_use,
'internal': self.internal})
def write(self):
"""
Write the metadata to Etcd
:return: None
"""
if self.cluster_type is None or self.cluster_type == '':
raise ValueError('Cluster type must be defined before being able to store the cluster metadata information')
etcd_key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
EtcdConfiguration.set(key=etcd_key, value={'type': self.cluster_type,
'in_use': self.in_use,
'internal': self.internal})
def ovs_3977_maintenance_agent_test():
"""
Test maintenance agent processes
"""
def _get_agent_distribution(agent_name):
result = {}
total = 0
for ip in alba_node_ips:
count = General.execute_command_on_node(ip, 'ls /etc/init/alba-maintenance_{0}-* | wc -l'.format(agent_name))
if count:
count = int(count)
else:
count = 0
total += count
result[ip] = count
result['total'] = total
print 'Maintenance agent distribution: {0}'.format(result)
for ip in alba_node_ips:
assert (result[ip] == total / len(alba_node_ips) or result[ip] == (total / len(alba_node_ips)) + 1),\
"Agents not equally distributed!"
return result
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
name = backend.alba_backend.name
alba_node_ips = [node.ip for node in GeneralAlba.get_alba_nodes()]
etcd_key = '/ovs/alba/backends/{0}/maintenance/nr_of_agents'.format(backend.alba_backend.guid)
nr_of_agents = EtcdConfiguration.get(etcd_key)
print '1. - nr of agents: {0}'.format(nr_of_agents)
actual_nr_of_agents = _get_agent_distribution(name)['total']
assert nr_of_agents == actual_nr_of_agents, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
# set nr to zero
EtcdConfiguration.set(etcd_key, 0)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 0, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '2. - nr of agents: {0}'.format(nr_of_agents)
# set nr to 10
EtcdConfiguration.set(etcd_key, 10)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 10, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '3. - nr of agents: {0}'.format(nr_of_agents)
def write_config(self):
"""
Writes the configuration down to in the format expected by Arakoon
"""
contents = RawConfigParser()
data = self.export()
for section in data:
contents.add_section(section)
for item in data[section]:
contents.set(section, item, data[section][item])
config_io = StringIO()
contents.write(config_io)
EtcdConfiguration.set(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), config_io.getvalue(), raw=True)
def write_config(self):
"""
Writes the configuration down to in the format expected by Arakoon
"""
contents = RawConfigParser()
data = self.export()
for section in data:
contents.add_section(section)
for item in data[section]:
contents.set(section, item, data[section][item])
config_io = StringIO()
contents.write(config_io)
EtcdConfiguration.set(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), config_io.getvalue(), raw=True)
def load(self):
"""
Loads the configuration from a given file, optionally a remote one
"""
self.configuration = {}
if EtcdConfiguration.dir_exists(self.path.format('')):
self.is_new = False
for key in self.params[self.config_type]:
if EtcdConfiguration.exists(self.path.format(key)):
self.configuration[key] = json.loads(EtcdConfiguration.get(self.path.format(key), raw=True))
else:
self._logger.debug('Could not find config {0}, a new one will be created'.format(self.path.format('')))
self.dirty_entries = []
def load(self):
"""
Loads the configuration from a given file, optionally a remote one
:param client: If provided, load remote configuration
"""
contents = '{}'
if EtcdConfiguration.exists(self.path):
contents = EtcdConfiguration.get(self.path, raw=True)
self.is_new = False
else:
logger.debug('Could not find config {0}, a new one will be created'.format(self.path))
self.dirty_entries = []
self.configuration = json.loads(contents)
def get_config(cluster_name):
"""
Retrieve the configuration for given cluster
:param cluster_name: Name of the cluster
:return: RawConfigParser object
"""
etcd_key = GeneralArakoon.ETCD_CONFIG_KEY.format(cluster_name)
if not EtcdConfiguration.exists(etcd_key, raw=True):
raise ValueError('Unknown arakoon cluster_name {0} provided'.format(cluster_name))
voldrv_config = EtcdConfiguration.get(etcd_key, raw=True)
parser = RawConfigParser()
parser.readfp(StringIO(voldrv_config))
return parser
def delete_cluster(cluster_name, ip):
"""
Deletes a complete cluster
:param ip: IP address of the last node of a cluster
:param cluster_name: Name of the cluster to remove
"""
logger.debug('Deleting cluster {0} on {1}'.format(cluster_name, ip))
config = ArakoonClusterConfig(cluster_name)
config.load_config()
# Cleans up a complete cluster (remove services, directories and configuration files)
for node in config.nodes:
ArakoonInstaller._destroy_node(config, node)
EtcdConfiguration.delete('{0}/{1}'.format(ArakoonInstaller.ETCD_CONFIG_ROOT, cluster_name), raw=True)
logger.debug('Deleting cluster {0} on {1} completed'.format(cluster_name, ip))
def collapse_arakoon():
"""
Collapse Arakoon's Tlogs
:return: None
"""
logger.info('Starting arakoon collapse')
arakoon_clusters = {}
for service in ServiceList.get_services():
if service.type.name in ('Arakoon', 'NamespaceManager', 'AlbaManager'):
arakoon_clusters[service.name.replace('arakoon-', '')] = service.storagerouter
for cluster, storagerouter in arakoon_clusters.iteritems():
logger.info(' Collapsing cluster {0}'.format(cluster))
contents = EtcdConfiguration.get(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(cluster), raw=True)
parser = RawConfigParser()
parser.readfp(StringIO(contents))
nodes = {}
for node in parser.get('global', 'cluster').split(','):
node = node.strip()
nodes[node] = ([parser.get(node, 'ip')], parser.get(node, 'client_port'))
config = ArakoonClientConfig(str(cluster), nodes)
for node in nodes.keys():
logger.info(' Collapsing node: {0}'.format(node))
client = ArakoonAdminClient(node, config)
try:
client.collapse_tlogs(2)
except:
logger.exception('Error during collapsing cluster {0} node {1}'.format(cluster, node))
logger.info('Arakoon collapse finished')
def load(self):
"""
Loads the configuration from a given file, optionally a remote one
"""
self.configuration = {}
if EtcdConfiguration.dir_exists(self.path.format('')):
self.is_new = False
for key in self.params[self.config_type]:
if EtcdConfiguration.exists(self.path.format(key)):
self.configuration[key] = json.loads(
EtcdConfiguration.get(self.path.format(key), raw=True))
else:
self._logger.debug(
'Could not find config {0}, a new one will be created'.format(
self.path.format('')))
self.dirty_entries = []
def load_config(self):
"""
Reads a configuration from reality
"""
contents = EtcdConfiguration.get(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), raw=True)
parser = RawConfigParser()
parser.readfp(StringIO(contents))
self.nodes = []
self._extra_globals = {}
for key in parser.options('global'):
if key == 'plugins':
self._plugins = [plugin.strip() for plugin in parser.get('global', 'plugins').split(',')]
elif key in ['cluster_id', 'cluster']:
pass # Ignore these
else:
self._extra_globals[key] = parser.get('global', key)
for node in parser.get('global', 'cluster').split(','):
node = node.strip()
self.nodes.append(ArakoonNodeConfig(name=node,
ip=parser.get(node, 'ip'),
client_port=parser.get(node, 'client_port'),
messaging_port=parser.get(node, 'messaging_port'),
log_sinks=parser.get(node, 'log_sinks'),
crash_log_sinks=parser.get(node, 'crash_log_sinks'),
home=parser.get(node, 'home'),
tlog_dir=parser.get(node, 'tlog_dir')))
def _configure_arakoon_to_volumedriver():
print "Update existing vPools"
logger.info("Update existing vPools")
for storagerouter in StorageRouterList.get_storagerouters():
config = ArakoonClusterConfig("voldrv")
config.load_config()
arakoon_nodes = []
for node in config.nodes:
arakoon_nodes.append({"host": node.ip, "port": node.client_port, "node_id": node.name})
with Remote(
storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], "ovs"
) as remote:
configuration_dir = "{0}/storagedriver/storagedriver".format(
EtcdConfiguration.get("/ovs/framework/paths|cfgdir")
)
if not remote.os.path.exists(configuration_dir):
remote.os.makedirs(configuration_dir)
for json_file in remote.os.listdir(configuration_dir):
vpool_name = json_file.replace(".json", "")
if json_file.endswith(".json"):
if remote.os.path.exists("{0}/{1}.cfg".format(configuration_dir, vpool_name)):
continue # There's also a .cfg file, so this is an alba_proxy configuration file
storagedriver_config = remote.StorageDriverConfiguration("storagedriver", vpool_name)
storagedriver_config.load()
storagedriver_config.configure_volume_registry(
vregistry_arakoon_cluster_id="voldrv", vregistry_arakoon_cluster_nodes=arakoon_nodes
)
storagedriver_config.configure_distributed_lock_store(
dls_type="Arakoon", dls_arakoon_cluster_id="voldrv", dls_arakoon_cluster_nodes=arakoon_nodes
)
storagedriver_config.save(reload_config=True)
def write_config(self):
"""
Writes the configuration down to in the format expected by Arakoon
"""
(temp_handle, temp_filename) = tempfile.mkstemp()
contents = RawConfigParser()
data = self.export()
for section in data:
contents.add_section(section)
for item in data[section]:
contents.set(section, item, data[section][item])
with open(temp_filename, 'wb') as config_file:
contents.write(config_file)
with open(temp_filename, 'r') as the_file:
EtcdConfiguration.set(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), the_file.read(), raw=True)
os.remove(temp_filename)
def collapse_arakoon():
"""
Collapse Arakoon's Tlogs
:return: None
"""
ScheduledTaskController._logger.info('Starting arakoon collapse')
arakoon_clusters = []
for service in ServiceList.get_services():
if service.is_internal is True and \
service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON,
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR):
arakoon_clusters.append(service.name.replace('arakoon-', ''))
for cluster in arakoon_clusters:
ScheduledTaskController._logger.info(' Collapsing cluster {0}'.format(cluster))
contents = EtcdConfiguration.get(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(cluster), raw=True)
parser = RawConfigParser()
parser.readfp(StringIO(contents))
nodes = {}
for node in parser.get('global', 'cluster').split(','):
node = node.strip()
nodes[node] = ([str(parser.get(node, 'ip'))], int(parser.get(node, 'client_port')))
config = ArakoonClientConfig(str(cluster), nodes)
for node in nodes.keys():
ScheduledTaskController._logger.info(' Collapsing node: {0}'.format(node))
client = ArakoonAdmin(config)
try:
client.collapse(str(node), 2)
except:
ScheduledTaskController._logger.exception('Error during collapsing cluster {0} node {1}'.format(cluster, node))
ScheduledTaskController._logger.info('Arakoon collapse finished')
def update_status(storagedriver_id):
"""
Sets Storage Driver offline in case hypervisor management Center
reports the hypervisor pmachine related to this Storage Driver
as unavailable.
:param storagedriver_id: ID of the storagedriver to update its status
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
storagerouter = storagedriver.storagerouter
if pmachine.mgmtcenter:
# Update status
pmachine.invalidate_dynamics(['host_status'])
else:
# No management Center, cannot update status via api
logger.info('Updating status of pmachine {0} using SSHClient'.format(pmachine.name))
host_status = 'RUNNING'
try:
client = SSHClient(storagerouter, username='******')
configuration_dir = EtcdConfiguration.get('/ovs/framework/paths|cfgdir')
logger.info('SSHClient connected successfully to {0} at {1}'.format(pmachine.name, client.ip))
with Remote(client.ip, [LocalStorageRouterClient]) as remote:
lsrc = remote.LocalStorageRouterClient('{0}/storagedriver/storagedriver/{1}.json'.format(configuration_dir,
storagedriver.vpool.name))
lsrc.server_revision()
logger.info('LocalStorageRouterClient connected successfully to {0} at {1}'.format(pmachine.name, client.ip))
except Exception as ex:
logger.error('Connectivity check failed, assuming host {0} is halted. {1}'.format(pmachine.name, ex))
host_status = 'HALTED'
if host_status != 'RUNNING':
# Host is stopped
storagedriver_client = StorageDriverClient.load(storagedriver.vpool)
storagedriver_client.mark_node_offline(str(storagedriver.storagedriver_id))
def load_config(self):
"""
Reads a configuration from reality
"""
contents = EtcdConfiguration.get(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), raw=True)
parser = RawConfigParser()
parser.readfp(StringIO(contents))
self.nodes = []
self._extra_globals = {}
for key in parser.options('global'):
if key == 'plugins':
self._plugins = [plugin.strip() for plugin in parser.get('global', 'plugins').split(',')]
elif key in ['cluster_id', 'cluster']:
pass # Ignore these
else:
self._extra_globals[key] = parser.get('global', key)
for node in parser.get('global', 'cluster').split(','):
node = node.strip()
self.nodes.append(ArakoonNodeConfig(name=node,
ip=parser.get(node, 'ip'),
client_port=parser.get(node, 'client_port'),
messaging_port=parser.get(node, 'messaging_port'),
log_dir=parser.get(node, 'log_dir'),
home=parser.get(node, 'home'),
tlog_dir=parser.get(node, 'tlog_dir')))
def load_target_definition(source, allow_override=False):
logging_target = {'type': 'console'}
try:
from ovs.extensions.db.etcd.configuration import EtcdConfiguration
logging_target = EtcdConfiguration.get('/ovs/framework/logging')
except:
pass
target_type = logging_target.get('type', 'console')
if allow_override is True and 'OVS_LOGTYPE_OVERRIDE' in os.environ:
target_type = os.environ['OVS_LOGTYPE_OVERRIDE']
if target_type == 'redis':
queue = logging_target.get('queue', '/ovs/logging')
if '{0}' in queue:
queue = queue.format(source)
return {
'type': 'redis',
'queue': '/{0}'.format(queue.lstrip('/')),
'host': logging_target.get('host', 'localhost'),
'port': logging_target.get('port', 6379)
}
if target_type == 'file':
return {'type': 'file', 'filename': LogHandler.load_path(source)}
return {'type': 'console'}
def _configure_arakoon_to_volumedriver(offline_node_ips=None):
print 'Update existing vPools'
logger.info('Update existing vPools')
if offline_node_ips is None:
offline_node_ips = []
for storagerouter in StorageRouterList.get_storagerouters():
config = ArakoonClusterConfig('voldrv')
config.load_config()
arakoon_nodes = []
for node in config.nodes:
arakoon_nodes.append({'host': node.ip,
'port': node.client_port,
'node_id': node.name})
with Remote(storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], 'ovs') as remote:
configuration_dir = '{0}/storagedriver/storagedriver'.format(EtcdConfiguration.get('/ovs/framework/paths|cfgdir'))
if not remote.os.path.exists(configuration_dir):
remote.os.makedirs(configuration_dir)
for json_file in remote.os.listdir(configuration_dir):
vpool_name = json_file.replace('.json', '')
if json_file.endswith('.json'):
if remote.os.path.exists('{0}/{1}.cfg'.format(configuration_dir, vpool_name)):
continue # There's also a .cfg file, so this is an alba_proxy configuration file
storagedriver_config = remote.StorageDriverConfiguration('storagedriver', vpool_name)
storagedriver_config.load()
storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id='voldrv',
vregistry_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon',
dls_arakoon_cluster_id='voldrv',
dls_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.save(reload_config=True)
def config_files_check_test():
"""
Verify some configuration files
"""
issues_found = ''
etcd_keys = {
"/ovs/framework/memcache",
"/ovs/arakoon/ovsdb/config"
}
for key_to_check in etcd_keys:
if not EtcdConfiguration.exists(key_to_check, raw = True):
issues_found += "Couldn't find {0}\n".format(key_to_check)
config_files = {
"rabbitmq.config": "/etc/rabbitmq/rabbitmq.config",
}
grid_ip = General.get_config().get('main', 'grid_ip')
ssh_pass = General.get_config().get('mgmtcenter', 'password')
client = SSHClient(grid_ip, username='******', password=ssh_pass)
for config_file_to_check in config_files.iterkeys():
if not client.file_exists(config_files[config_file_to_check]):
issues_found += "Couldn't find {0}\n".format(config_file_to_check)
assert issues_found == '', "Found the following issues while checking for the config files:{0}\n".format(issues_found)
def __init__(self, config_type, vpool_name, number=None):
"""
Initializes the class
"""
def make_configure(sct):
"""
section closure
:param sct: Section to create configure function for
"""
return lambda **kwargs: self._add(sct, **kwargs)
if config_type not in ['storagedriver', 'metadataserver']:
raise RuntimeError('Invalid configuration type. Allowed: storagedriver, metadataserver')
self.config_type = config_type
self.vpool_name = vpool_name
self.configuration = {}
self.is_new = True
self.dirty_entries = []
self.number = number
self.params = copy.deepcopy(StorageDriverConfiguration.parameters) # Never use parameters directly
self.base_path = '{0}/storagedriver/{1}'.format(EtcdConfiguration.get('/ovs/framework/paths|cfgdir'), self.config_type)
if self.number is None:
self.path = '{0}/{1}.json'.format(self.base_path, self.vpool_name)
else:
self.path = '{0}/{1}_{2}.json'.format(self.base_path, self.vpool_name, self.number)
# Fix some manual "I know what I'm doing" overrides
backend_connection_manager = 'backend_connection_manager'
self.params[self.config_type][backend_connection_manager]['optional'].append('s3_connection_strict_consistency')
# Generate configure_* methods
for section in self.params[self.config_type]:
setattr(self, 'configure_{0}'.format(section), make_configure(section))
def _has_plugin(self):
"""
Checks whether this BackendType has a plugin installed
"""
try:
return self.code in EtcdConfiguration.get('/ovs/framework/plugins/installed|backends')
except:
return False
def get_path(binary_name):
"""
Retrieve the absolute path for binary
:param binary_name: Binary to get path for
:return: Path
"""
machine_id = System.get_my_machine_id()
config_location = '/ovs/framework/hosts/{0}/paths|{1}'.format(machine_id, binary_name)
if not EtcdConfiguration.exists(config_location):
try:
path = check_output('which {0}'.format(binary_name), shell=True).strip()
EtcdConfiguration.set(config_location, path)
except CalledProcessError:
return None
else:
path = EtcdConfiguration.get(config_location)
return path
def delete_cluster(cluster_name, ip):
"""
Deletes a complete cluster
:param ip: IP address of the last node of a cluster
:param cluster_name: Name of the cluster to remove
"""
logger.debug('Deleting cluster {0} on {1}'.format(cluster_name, ip))
config = ArakoonClusterConfig(cluster_name)
config.load_config()
# Cleans up a complete cluster (remove services, directories and configuration files)
for node in config.nodes:
ArakoonInstaller._destroy_node(config, node)
EtcdConfiguration.delete('{0}/{1}'.format(
ArakoonInstaller.ETCD_CONFIG_ROOT, cluster_name),
raw=True)
logger.debug('Deleting cluster {0} on {1} completed'.format(
cluster_name, ip))
def get_arakoon_metadata_by_cluster_name(cluster_name):
"""
Retrieve arakoon cluster information based on its name
:param cluster_name: Name of the arakoon cluster
:type cluster_name: str
:return: Arakoon cluster metadata information
:rtype: ArakoonClusterMetadata
"""
if not EtcdConfiguration.exists('/ovs/arakoon', raw=True):
raise ValueError('Etcd key "/ovs/arakoon" not found')
for cluster in EtcdConfiguration.list('/ovs/arakoon'):
if cluster == cluster_name:
arakoon_metadata = ArakoonClusterMetadata(cluster_id=cluster_name)
arakoon_metadata.load_metadata()
return arakoon_metadata
raise ValueError('No arakoon cluster found with name "{0}"'.format(cluster_name))
def _has_plugin(self):
"""
Checks whether this BackendType has a plugin installed
"""
try:
return self.code in EtcdConfiguration.get(
'/ovs/framework/plugins/installed|backends')
except:
return False
def save(self, client=None, reload_config=True):
"""
Saves the configuration to a given file, optionally a remote one
:param client: If provided, save remote configuration
:param reload_config: Reload the running Storage Driver configuration
"""
self._validate()
for key in self.configuration:
contents = json.dumps(self.configuration[key], indent=4)
EtcdConfiguration.set(self.path.format(key), contents, raw=True)
if self.config_type == 'storagedriver' and reload_config is True:
if len(self.dirty_entries) > 0:
if client is None:
self._logger.info(
'Applying local storagedriver configuration changes')
changes = LSRClient(self.remote_path).update_configuration(
self.remote_path)
else:
self._logger.info(
'Applying storagedriver configuration changes on {0}'.
format(client.ip))
with remote(client.ip, [LSRClient]) as rem:
changes = copy.deepcopy(
rem.LocalStorageRouterClient(
self.remote_path).update_configuration(
self.remote_path))
for change in changes:
if change['param_name'] not in self.dirty_entries:
raise RuntimeError(
'Unexpected configuration change: {0}'.format(
change['param_name']))
self._logger.info('Changed {0} from "{1}" to "{2}"'.format(
change['param_name'], change['old_value'],
change['new_value']))
self.dirty_entries.remove(change['param_name'])
self._logger.info('Changes applied')
if len(self.dirty_entries) > 0:
self._logger.warning(
'Following changes were not applied: {0}'.format(
', '.join(self.dirty_entries)))
else:
self._logger.debug('No need to apply changes, nothing changed')
self.is_new = False
self.dirty_entries = []
def migrate(master_ips=None, extra_ips=None):
"""
Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one
:param master_ips: IP addresses of the MASTER nodes
:param extra_ips: IP addresses of the EXTRA nodes
"""
machine_id = System.get_my_machine_id()
key = '/ovs/framework/hosts/{0}/versions'.format(machine_id)
try:
data = EtcdConfiguration.get(key) if EtcdConfiguration.exists(
key) else {}
except EtcdConnectionFailed:
import json # Most likely 2.6 to 2.7 migration
data = {}
filename = '/opt/OpenvStorage/config/ovs.json'
if os.path.exists(filename):
with open(filename) as config_file:
data = json.load(config_file).get('core',
{}).get('versions', {})
migrators = []
path = '/'.join([os.path.dirname(__file__), 'migration'])
for filename in os.listdir(path):
if os.path.isfile('/'.join([path, filename
])) and filename.endswith('.py'):
name = filename.replace('.py', '')
module = imp.load_source(name, '/'.join([path, filename]))
for member in inspect.getmembers(module):
if inspect.isclass(
member[1]
) and member[1].__module__ == name and 'object' in [
base.__name__ for base in member[1].__bases__
]:
migrators.append(
(member[1].identifier, member[1].migrate))
end_version = 0
for identifier, method in migrators:
base_version = data[identifier] if identifier in data else 0
version = method(base_version, master_ips, extra_ips)
if version > end_version:
end_version = version
data[identifier] = end_version
EtcdConfiguration.set(key, data)
def get_path(binary_name):
"""
Retrieve the absolute path for binary
:param binary_name: Binary to get path for
:return: Path
"""
machine_id = System.get_my_machine_id()
config_location = '/ovs/framework/hosts/{0}/paths|{1}'.format(
machine_id, binary_name)
if not EtcdConfiguration.exists(config_location):
try:
path = check_output('which {0}'.format(binary_name),
shell=True).strip()
EtcdConfiguration.set(config_location, path)
except CalledProcessError:
return None
else:
path = EtcdConfiguration.get(config_location)
return path
def write_config(self):
"""
Writes the configuration down to in the format expected by Arakoon
"""
(temp_handle, temp_filename) = tempfile.mkstemp()
contents = RawConfigParser()
data = self.export()
for section in data:
contents.add_section(section)
for item in data[section]:
contents.set(section, item, data[section][item])
with open(temp_filename, 'wb') as config_file:
contents.write(config_file)
with open(temp_filename, 'r') as the_file:
EtcdConfiguration.set(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(
self.cluster_id),
the_file.read(),
raw=True)
os.remove(temp_filename)
def pulse():
"""
Update the heartbeats for all Storage Routers
:return: None
"""
logger = LogHandler.get('extensions', name='heartbeat')
current_time = int(time.time())
machine_id = System.get_my_machine_id()
amqp = '{0}://{1}:{2}@{3}//'.format(
EtcdConfiguration.get('/ovs/framework/messagequeue|protocol'),
EtcdConfiguration.get('/ovs/framework/messagequeue|user'),
EtcdConfiguration.get('/ovs/framework/messagequeue|password'),
EtcdConfiguration.get(
'/ovs/framework/hosts/{0}/ip'.format(machine_id)))
celery_path = OSManager.get_path('celery')
worker_states = check_output(
"{0} inspect ping -b {1} --timeout=5 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true"
.format(celery_path, amqp),
shell=True)
routers = StorageRouterList.get_storagerouters()
for node in routers:
if node.heartbeats is None:
node.heartbeats = {}
if 'celery@{0}: OK'.format(node.name) in worker_states:
node.heartbeats['celery'] = current_time
if node.machine_id == machine_id:
node.heartbeats['process'] = current_time
else:
try:
# check timeout of other nodes and clear arp cache
if node.heartbeats and 'process' in node.heartbeats:
if current_time - node.heartbeats[
'process'] >= HeartBeat.ARP_TIMEOUT:
check_output("/usr/sbin/arp -d {0}".format(
node.name),
shell=True)
except CalledProcessError:
logger.exception('Error clearing ARP cache')
node.save()
def _create_unit(fleet_name, template_file):
from ovs.extensions.db.etcd.configuration import EtcdConfiguration
start = time.time()
while time.time() - start < 60:
try:
unit = FLEET_CLIENT.create_unit(
fleet_name, fleet.Unit(from_string=template_file))
return unit
except fleet.APIError as ae:
if ae.code == 500:
FleetCtl._logger.warning(
'API Error in fleet, most likely caused by etcd, retrying. {0}'
.format(ae))
key = '/_coreos.com/fleet/job/{0}/object'.format(
fleet_name)
if EtcdConfiguration.exists(key):
EtcdConfiguration.delete(key)
time.sleep(1)
else:
raise
raise RuntimeError('Failed to create ')
def get_client(client_type=None):
"""
Returns a volatile storage client
"""
if not hasattr(VolatileFactory, 'store') or VolatileFactory.store is None:
if hasattr(unittest, 'running_tests') and getattr(unittest, 'running_tests'):
client_type = 'dummy'
if client_type is None:
client_type = EtcdConfiguration.get('/ovs/framework/stores|volatile')
VolatileFactory.store = None
if client_type == 'memcache':
from ovs.extensions.storage.volatile.memcachestore import MemcacheStore
nodes = EtcdConfiguration.get('/ovs/framework/memcache|endpoints')
VolatileFactory.store = MemcacheStore(nodes)
if client_type == 'dummy':
from ovs.extensions.storage.volatile.dummystore import DummyVolatileStore
VolatileFactory.store = DummyVolatileStore()
if VolatileFactory.store is None:
raise RuntimeError('Invalid client_type specified')
return VolatileFactory.store
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:type cluster_ip: str
:param master_ip: IP of the master node
:type master_ip: str
:param offline_node_ips: IPs of nodes which are offline
:type offline_node_ips: list
:return: None
"""
_ = master_ip
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(
cluster_ip,
username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv' and service.is_internal is True: # Externally managed arakoon cluster service does not have storage router
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
StorageDriverController._logger.debug(
'* Shrink StorageDriver cluster')
cluster_name = str(
EtcdConfiguration.get(
'/ovs/framework/arakoon_clusters|voldrv'))
ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip,
cluster_name=cluster_name,
offline_nodes=offline_node_ips)
if client is not None and ServiceManager.has_service(
current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name,
client=client)
ServiceManager.remove_service(current_service.name,
client=client)
ArakoonInstaller.restart_cluster_remove(cluster_name,
remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
def _process_task(task, metadata, servicemanager):
"""
Processes a task
"""
try:
logger.debug('Processing: {0}'.format(task))
cid = EtcdConfiguration.get('/ovs/framework/cluster_id')
nid = System.get_my_machine_id()
if task == 'OPEN_TUNNEL':
if servicemanager == 'upstart':
check_output('service openvpn stop', shell=True)
else:
check_output(
'systemctl stop openvpn@ovs_{0}-{1} || true'.format(
cid, nid),
shell=True)
check_output('rm -f /etc/openvpn/ovs_*', shell=True)
for filename, contents in metadata['files'].iteritems():
with open(filename, 'w') as the_file:
the_file.write(base64.b64decode(contents))
if servicemanager == 'upstart':
check_output('service openvpn start', shell=True)
else:
check_output('systemctl start openvpn@ovs_{0}-{1}'.format(
cid, nid),
shell=True)
elif task == 'CLOSE_TUNNEL':
if servicemanager == 'upstart':
check_output('service openvpn stop', shell=True)
else:
check_output('systemctl stop openvpn@ovs_{0}-{1}'.format(
cid, nid),
shell=True)
check_output('rm -f /etc/openvpn/ovs_*', shell=True)
elif task == 'UPLOAD_LOGFILES':
logfile = check_output('ovs collect logs', shell=True).strip()
check_output(
'mv {0} /tmp/{1}; curl -T /tmp/{1} ftp://{2} --user {3}:{4}; rm -f {0} /tmp/{1}'
.format(logfile, metadata['filename'],
metadata['endpoint'], metadata['user'],
metadata['password']),
shell=True)
else:
raise RuntimeError('Unknown task')
except Exception, ex:
logger.exception(
'Unexpected error while processing task {0} (data: {1}): {2}'.
format(task, json.dumps(metadata), ex))
raise
def __init__(self):
"""
Initializes the client
"""
self._enable_support = EtcdConfiguration.get(
'/ovs/framework/support|enablesupport')
self.interval = EtcdConfiguration.get(
'/ovs/framework/support|interval')
self._url = 'https://monitoring.openvstorage.com/api/support/heartbeat/'
init_info = check_output('cat /proc/1/comm', shell=True)
# All service classes used in below code should share the exact same interface!
if 'init' in init_info:
version_info = check_output('init --version', shell=True)
if 'upstart' in version_info:
self.servicemanager = 'upstart'
else:
RuntimeError(
'There was no known service manager detected in /proc/1/comm'
)
elif 'systemd' in init_info:
self.servicemanager = 'systemd'
else:
raise RuntimeError(
'There was no known service manager detected in /proc/1/comm')
def get_heartbeat_data(self):
"""
Returns heartbeat data
"""
data = {
'cid': EtcdConfiguration.get('/ovs/framework/cluster_id'),
'nid': System.get_my_machine_id(),
'metadata': {},
'errors': []
}
try:
# Versions
data['metadata']['versions'] = PackageManager.get_versions()
except Exception, ex:
data['errors'].append(str(ex))
def update_status(storagedriver_id):
"""
Sets Storage Driver offline in case hypervisor management Center
reports the hypervisor pmachine related to this Storage Driver
as unavailable.
:param storagedriver_id: ID of the storagedriver to update its status
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
storagerouter = storagedriver.storagerouter
if pmachine.mgmtcenter:
# Update status
pmachine.invalidate_dynamics(['host_status'])
else:
# No management Center, cannot update status via api
logger.info(
'Updating status of pmachine {0} using SSHClient'.format(
pmachine.name))
host_status = 'RUNNING'
try:
client = SSHClient(storagerouter, username='******')
configuration_dir = EtcdConfiguration.get(
'/ovs/framework/paths|cfgdir')
logger.info(
'SSHClient connected successfully to {0} at {1}'.format(
pmachine.name, client.ip))
with Remote(client.ip, [LocalStorageRouterClient]) as remote:
lsrc = remote.LocalStorageRouterClient(
'{0}/storagedriver/storagedriver/{1}.json'.format(
configuration_dir, storagedriver.vpool.name))
lsrc.server_revision()
logger.info(
'LocalStorageRouterClient connected successfully to {0} at {1}'
.format(pmachine.name, client.ip))
except Exception as ex:
logger.error(
'Connectivity check failed, assuming host {0} is halted. {1}'
.format(pmachine.name, ex))
host_status = 'HALTED'
if host_status != 'RUNNING':
# Host is stopped
storagedriver_client = StorageDriverClient.load(
storagedriver.vpool)
storagedriver_client.mark_node_offline(
str(storagedriver.storagedriver_id))
