def remove_mds_service(mds_service, vpool, reload_config):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:param vpool: The vPool for which the MDS service will be removed
:param reload_config: If True, the volumedriver's updated configuration will be reloaded
"""
if len(mds_service.vdisks_guids) > 0:
raise RuntimeError('Cannot remove MDSService that is still serving disks')
storagerouter = mds_service.service.storagerouter
client = SSHClient(storagerouter)
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
# Clean up model
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
mds_service.delete()
mds_service.service.delete()
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
sdp = [sd_partition.path for sd_partition in mds_service.storagedriver_partitions if sd_partition.role == DiskPartition.ROLES.DB]
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp[0],
'scratch_directory': sdp[0]})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
tries = 5
while tries > 0:
try:
root_client = SSHClient(storagerouter, username='******')
root_client.dir_delete(directories=directories_to_clean,
follow_symlinks=True)
for dir_name in directories_to_clean:
logger.debug('Recursively removed {0}'.format(dir_name))
break
except Exception:
time.sleep(5)
logger.debug('Waiting for the MDS service to go down...')
tries -= 1
if tries == 0:
raise
def remove_mds_service(mds_service, client, storagerouter, vpool, reload_config):
"""
Removes an MDS service
"""
if len(mds_service.vdisks_guids) > 0:
raise RuntimeError('Cannot remove MDSService that is still serving disks')
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid][0]
# Clean up model
this_service_number = mds_service.number
service = mds_service.service
mds_service.delete()
service.delete()
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_md,
vpool.name,
mds_service.number),
'scratch_directory': '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_temp,
vpool.name,
mds_service.number)})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
tries = 5
cleaned = False
while tries > 0 and cleaned is False:
try:
client.dir_delete(['{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_md,
vpool.name,
this_service_number),
'{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_temp,
vpool.name,
this_service_number)])
logger.debug('MDS files cleaned up')
cleaned = True
except Exception:
time.sleep(5)
logger.debug('Waiting for the MDS service to go down...')
tries -= 1
def remove_mds_service(mds_service, client, storagerouter, vpool, reload_config):
"""
Removes an MDS service
"""
if len(mds_service.vdisks_guids) > 0:
raise RuntimeError('Cannot remove MDSService that is still serving disks')
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
# Clean up model
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
service = mds_service.service
mds_service.delete()
service.delete()
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': [sd_partition.path for sd_partition in mds_service.storagedriver_partitions if sd_partition.role == DiskPartition.ROLES.DB][0],
'scratch_directory': [sd_partition.path for sd_partition in mds_service.storagedriver_partitions if sd_partition.role == DiskPartition.ROLES.SCRUB][0]})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
tries = 5
cleaned = False
while tries > 0 and cleaned is False:
try:
client.dir_delete(directories_to_clean)
logger.debug('MDS files cleaned up')
cleaned = True
except Exception:
time.sleep(5)
logger.debug('Waiting for the MDS service to go down...')
tries -= 1
def prepare_mds_service(storagerouter, vpool, fresh_only, reload_config):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
:param storagerouter: Storagerouter on which MDS service will be created
:type storagerouter: StorageRouter
:param vpool: The vPool for which the MDS service will be created
:type vpool: VPool
:param fresh_only: If True and no current mds services exist for this vpool on this storagerouter, a new 1 will be created
:type fresh_only: bool
:param reload_config: If True, the volumedriver's updated configuration will be reloaded
:type reload_config: bool
:return: Newly created service
:rtype: MDSService
"""
# Fetch service sequence number based on MDS services for current vPool and current storage router
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return # There is already 1 or more MDS services running, aborting
# VALIDATIONS
# 1. Find free port based on MDS services for all vPools on current storage router
client = SSHClient(storagerouter)
mdsservice_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.MD_SERVER)
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.extend(service.ports)
mds_port_range = Configuration.get(
'/ovs/framework/hosts/{0}/ports|mds'.format(
System.get_my_machine_id(client)))
free_ports = System.get_free_ports(selected_range=mds_port_range,
exclude=occupied_ports,
nr=1,
client=client)
if not free_ports:
raise RuntimeError(
'Failed to find an available port on storage router {0} within range {1}'
.format(storagerouter.name, mds_port_range))
# 2. Partition check
db_partition = None
for disk in storagerouter.disks:
for partition in disk.partitions:
if DiskPartition.ROLES.DB in partition.roles:
db_partition = partition
break
if db_partition is None:
raise RuntimeError(
'Could not find DB partition on storage router {0}'.format(
storagerouter.name))
# 3. Verify storage driver configured
storagedrivers = [
sd for sd in vpool.storagedrivers
if sd.storagerouter_guid == storagerouter.guid
]
if not storagedrivers:
raise RuntimeError(
'Expected to find a configured storagedriver for vpool {0} on storage router {1}'
.format(vpool.name, storagerouter.name))
storagedriver = storagedrivers[0]
# MODEL UPDATES
# 1. Service
service_number += 1
service = Service()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name,
service_number)
service.type = mdsservice_type
service.ports = [free_ports[0]]
service.storagerouter = storagerouter
service.save()
mds_service = MDSService()
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.service = service
mds_service.save()
# 2. Storage driver partitions
from ovs.lib.storagedriver import StorageDriverController
StorageDriverController.add_storagedriverpartition(
storagedriver, {
'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': db_partition,
'mds_service': mds_service
})
# CONFIGURATIONS
# 1. Volumedriver
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service is not None:
if mds_service.vpool_guid == vpool.guid:
sdp = [
sd_partition for sd_partition in
mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.DB
and sd_partition.sub_role ==
StorageDriverPartition.SUBROLE.MDS
][0]
mds_nodes.append({
'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp.path,
'scratch_directory': sdp.path
})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration(
'storagedriver', vpool.guid, storagedriver.storagedriver_id)
storagedriver_config.load()
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
return mds_service
def remove_mds_service(mds_service,
vpool,
reconfigure,
allow_offline=False):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:type mds_service: MDSService
:param vpool: The vPool for which the MDS service will be removed
:type vpool: VPool
:param reconfigure: Indicates whether reconfiguration is required
:type reconfigure: bool
:param allow_offline: Indicates whether it's OK that the node for which mds services are cleaned is offline
:type allow_offline: bool
"""
if len(mds_service.vdisks_guids) > 0 and allow_offline is False:
raise RuntimeError(
'Cannot remove MDSService that is still serving disks')
mdsservice_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.MD_SERVER)
# Clean up model
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
if allow_offline is True: # Certain vdisks might still be attached to this offline MDS service --> Delete relations
for junction in mds_service.vdisks:
junction.delete()
mds_service.delete()
mds_service.service.delete()
storagerouter = mds_service.service.storagerouter
try:
client = SSHClient(storagerouter)
if reconfigure is True:
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
sdp = [
sd_partition for sd_partition in
mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.DB
and sd_partition.sub_role ==
StorageDriverPartition.SUBROLE.MDS
][0]
mds_nodes.append({
'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp.path,
'scratch_directory': sdp.path
})
# Generate the correct section in the Storage Driver's configuration
storagedriver = [
sd for sd in storagerouter.storagedrivers
if sd.vpool_guid == vpool.guid
][0]
storagedriver_config = StorageDriverConfiguration(
'storagedriver', vpool.guid,
storagedriver.storagedriver_id)
storagedriver_config.load()
storagedriver_config.configure_metadata_server(
mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reconfigure)
tries = 5
while tries > 0:
try:
root_client = SSHClient(storagerouter, username='******')
root_client.dir_delete(directories=directories_to_clean,
follow_symlinks=True)
for dir_name in directories_to_clean:
MDSServiceController._logger.debug(
'Recursively removed {0}'.format(dir_name))
break
except Exception:
MDSServiceController._logger.debug(
'Waiting for the MDS service to go down...')
time.sleep(5)
tries -= 1
if tries == 0:
raise
except UnableToConnectException:
if allow_offline is True:
MDSServiceController._logger.info(
'Allowed offline node during mds service removal')
else:
raise
def remove_mds_service(mds_service, reconfigure, allow_offline=False):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:type mds_service: ovs.dal.hybrids.j_mdsservice.MDSService
:param reconfigure: Indicates whether reconfiguration is required
:type reconfigure: bool
:param allow_offline: Indicates whether it's OK that the node for which mds services are cleaned is offline
:type allow_offline: bool
:raises RuntimeError: When vDisks present on the MDSService to be removed
No StorageDriver is linked to the MDSService to be removed
:raises UnableToConnectException: When StorageRouter on which the MDSService resides is unreachable and allow_offline flag is False
:return: None
:rtype: NoneType
"""
if len(mds_service.vdisks_guids) > 0 and allow_offline is False:
raise RuntimeError(
'Cannot remove MDSService that is still serving disks')
if len(
mds_service.storagedriver_partitions
) == 0 or mds_service.storagedriver_partitions[0].storagedriver is None:
raise RuntimeError(
'Failed to retrieve the linked StorageDriver to this MDS Service {0}'
.format(mds_service.service.name))
vpool = mds_service.vpool
root_client = None
storagerouter = mds_service.service.storagerouter
storagedriver = mds_service.storagedriver_partitions[0].storagedriver
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Removing MDS junction service for port {2}'
.format(storagerouter.name, vpool.name,
mds_service.service.ports[0]))
try:
root_client = SSHClient(endpoint=storagerouter, username='******')
MDSServiceController._logger.debug(
'StorageRouter {0} - vPool {1}: Established SSH connection'.
format(storagerouter.name, vpool.name))
except UnableToConnectException:
if allow_offline is True:
MDSServiceController._logger.warning(
'StorageRouter {0} - vPool {1}: Allowed offline node during MDS service removal'
.format(storagerouter.name, vpool.name))
else:
MDSServiceController._logger.exception(
'StorageRouter {0} - vPool {1}: Failed to connect to StorageRouter'
.format(storagerouter.name, vpool.name))
raise
# Reconfigure StorageDriver
if reconfigure is True and root_client is not None:
mds_nodes = []
for sd_partition in storagedriver.partitions:
if sd_partition.role == DiskPartition.ROLES.DB and sd_partition.sub_role == StorageDriverPartition.SUBROLE.MDS and sd_partition.mds_service != mds_service:
service = sd_partition.mds_service.service
mds_nodes.append({
'host':
service.storagerouter.ip,
'port':
service.ports[0],
'db_directory':
'{0}/db'.format(sd_partition.path),
'scratch_directory':
'{0}/scratch'.format(sd_partition.path)
})
# Generate the correct section in the StorageDriver's configuration
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Configuring StorageDriver with MDS nodes: {2}'
.format(storagerouter.name, vpool.name, mds_nodes))
storagedriver_config = StorageDriverConfiguration(
vpool.guid, storagedriver.storagedriver_id)
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(root_client)
# Clean up model
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Cleaning model'.format(
storagerouter.name, vpool.name))
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
if allow_offline is True: # Certain vDisks might still be attached to this offline MDS service --> Delete relations
for junction in mds_service.vdisks:
junction.delete()
mds_service.delete()
mds_service.service.delete()
# Clean up file system
if root_client is not None:
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Deleting directories from file system: {2}'
.format(storagerouter.name, vpool.name, directories_to_clean))
tries = 5
while tries > 0:
try:
root_client.dir_delete(directories=directories_to_clean,
follow_symlinks=True)
for dir_name in directories_to_clean:
MDSServiceController._logger.debug(
'StorageRouter {0} - vPool {1}: Recursively removed directory: {2}'
.format(storagerouter.name, vpool.name, dir_name))
break
except Exception:
MDSServiceController._logger.warning(
'StorageRouter {0} - vPool {1}: Waiting for the MDS service to go down...'
.format(storagerouter.name, vpool.name))
time.sleep(5)
tries -= 1
if tries == 0:
MDSServiceController._logger.exception(
'StorageRouter {0} - vPool {1}: Deleting directories failed'
.format(storagerouter.name, vpool.name))
raise
def prepare_mds_service(storagerouter, vpool, fresh_only, reload_config):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
:param storagerouter: Storagerouter on which MDS service will be created
:param vpool: The vPool for which the MDS service will be created
:param fresh_only: If True and no current mds services exist for this vpool on this storagerouter, a new 1 will be created
:param reload_config: If True, the volumedriver's updated configuration will be reloaded
"""
# Fetch service sequence number based on MDS services for current vPool and current storage router
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return # There is already 1 or more MDS services running, aborting
# VALIDATIONS
# 1. Find free port based on MDS services for all vPools on current storage router
client = SSHClient(storagerouter)
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.extend(service.ports)
mds_port_range = client.config_read('ovs.ports.mds')
free_ports = System.get_free_ports(selected_range=mds_port_range,
exclude=occupied_ports,
nr=1,
client=client)
if not free_ports:
raise RuntimeError('Failed to find an available port on storage router {0} within range {1}'.format(storagerouter.name, mds_port_range))
# 2. Partition check
db_partition = None
for disk in storagerouter.disks:
for partition in disk.partitions:
if DiskPartition.ROLES.DB in partition.roles:
db_partition = partition
break
if db_partition is None:
raise RuntimeError('Could not find DB partition on storage router {0}'.format(storagerouter.name))
# 3. Verify storage driver configured
storagedrivers = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid]
if not storagedrivers:
raise RuntimeError('Expected to find a configured storagedriver for vpool {0} on storage router {1}'.format(vpool.name, storagerouter.name))
# MODEL UPDATES
# 1. Service
service_number += 1
service = Service()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name, service_number)
service.type = mdsservice_type
service.ports = [free_ports[0]]
service.storagerouter = storagerouter
service.save()
mds_service = MDSService()
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.service = service
mds_service.save()
# 2. Storage driver partitions
from ovs.lib.storagedriver import StorageDriverController
sdp = StorageDriverController.add_storagedriverpartition(storagedrivers[0], {'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': db_partition,
'mds_service': mds_service})
# CONFIGURATIONS
# 1. Volumedriver
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp.path,
'scratch_directory': sdp.path})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
return mds_service
def remove_mds_service(mds_service, vpool, reconfigure, allow_offline=False):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:param vpool: The vPool for which the MDS service will be removed
:param reconfigure: Indicates whether reconfiguration is required
:param allow_offline: Indicates whether it's OK that the node for which mds services are cleaned is offline
"""
if len(mds_service.vdisks_guids) > 0 and allow_offline is False:
raise RuntimeError('Cannot remove MDSService that is still serving disks')
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
# Clean up model
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
if allow_offline is True: # Certain vdisks might still be attached to this offline MDS service --> Delete relations
for junction in mds_service.vdisks:
junction.delete()
mds_service.delete()
mds_service.service.delete()
storagerouter = mds_service.service.storagerouter
try:
client = SSHClient(storagerouter)
if reconfigure is True:
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
sdp = [sd_partition.path for sd_partition in mds_service.storagedriver_partitions if sd_partition.role == DiskPartition.ROLES.DB]
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp[0],
'scratch_directory': sdp[0]})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reconfigure)
tries = 5
while tries > 0:
try:
root_client = SSHClient(storagerouter, username='******')
root_client.dir_delete(directories=directories_to_clean,
follow_symlinks=True)
for dir_name in directories_to_clean:
logger.debug('Recursively removed {0}'.format(dir_name))
break
except Exception:
logger.debug('Waiting for the MDS service to go down...')
time.sleep(5)
tries -= 1
if tries == 0:
raise
except UnableToConnectException:
if allow_offline is True:
logger.info('Allowed offline node during mds service removal')
else:
raise
def prepare_mds_service(storagerouter, vpool, fresh_only, reload_config):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
:param storagerouter: Storagerouter on which MDS service will be created
:type storagerouter: StorageRouter
:param vpool: The vPool for which the MDS service will be created
:type vpool: VPool
:param fresh_only: If True and no current mds services exist for this vpool on this storagerouter, a new 1 will be created
:type fresh_only: bool
:param reload_config: If True, the volumedriver's updated configuration will be reloaded
:type reload_config: bool
:return: Newly created service
:rtype: MDSService
"""
# Fetch service sequence number based on MDS services for current vPool and current storage router
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return # There is already 1 or more MDS services running, aborting
# VALIDATIONS
# 1. Find free port based on MDS services for all vPools on current storage router
client = SSHClient(storagerouter)
mdsservice_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.MD_SERVER)
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.extend(service.ports)
mds_port_range = Configuration.get(
"/ovs/framework/hosts/{0}/ports|mds".format(System.get_my_machine_id(client))
)
free_ports = System.get_free_ports(selected_range=mds_port_range, exclude=occupied_ports, nr=1, client=client)
if not free_ports:
raise RuntimeError(
"Failed to find an available port on storage router {0} within range {1}".format(
storagerouter.name, mds_port_range
)
)
# 2. Partition check
db_partition = None
for disk in storagerouter.disks:
for partition in disk.partitions:
if DiskPartition.ROLES.DB in partition.roles:
db_partition = partition
break
if db_partition is None:
raise RuntimeError("Could not find DB partition on storage router {0}".format(storagerouter.name))
# 3. Verify storage driver configured
storagedrivers = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid]
if not storagedrivers:
raise RuntimeError(
"Expected to find a configured storagedriver for vpool {0} on storage router {1}".format(
vpool.name, storagerouter.name
)
)
storagedriver = storagedrivers[0]
# MODEL UPDATES
# 1. Service
service_number += 1
service = Service()
service.name = "metadataserver_{0}_{1}".format(vpool.name, service_number)
service.type = mdsservice_type
service.ports = [free_ports[0]]
service.storagerouter = storagerouter
service.save()
mds_service = MDSService()
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.service = service
mds_service.save()
# 2. Storage driver partitions
from ovs.lib.storagedriver import StorageDriverController
StorageDriverController.add_storagedriverpartition(
storagedriver,
{
"size": None,
"role": DiskPartition.ROLES.DB,
"sub_role": StorageDriverPartition.SUBROLE.MDS,
"partition": db_partition,
"mds_service": mds_service,
},
)
# CONFIGURATIONS
# 1. Volumedriver
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service is not None:
if mds_service.vpool_guid == vpool.guid:
sdp = [
sd_partition
for sd_partition in mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.DB
and sd_partition.sub_role == StorageDriverPartition.SUBROLE.MDS
][0]
mds_nodes.append(
{
"host": service.storagerouter.ip,
"port": service.ports[0],
"db_directory": sdp.path,
"scratch_directory": sdp.path,
}
)
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration("storagedriver", vpool.guid, storagedriver.storagedriver_id)
storagedriver_config.load()
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
return mds_service
def remove_mds_service(mds_service, vpool, reconfigure, allow_offline=False):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:type mds_service: MDSService
:param vpool: The vPool for which the MDS service will be removed
:type vpool: VPool
:param reconfigure: Indicates whether reconfiguration is required
:type reconfigure: bool
:param allow_offline: Indicates whether it's OK that the node for which mds services are cleaned is offline
:type allow_offline: bool
"""
if len(mds_service.vdisks_guids) > 0 and allow_offline is False:
raise RuntimeError("Cannot remove MDSService that is still serving disks")
mdsservice_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.MD_SERVER)
# Clean up model
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
if (
allow_offline is True
): # Certain vdisks might still be attached to this offline MDS service --> Delete relations
for junction in mds_service.vdisks:
junction.delete()
mds_service.delete()
mds_service.service.delete()
storagerouter = mds_service.service.storagerouter
try:
client = SSHClient(storagerouter)
if reconfigure is True:
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
sdp = [
sd_partition
for sd_partition in mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.DB
and sd_partition.sub_role == StorageDriverPartition.SUBROLE.MDS
][0]
mds_nodes.append(
{
"host": service.storagerouter.ip,
"port": service.ports[0],
"db_directory": sdp.path,
"scratch_directory": sdp.path,
}
)
# Generate the correct section in the Storage Driver's configuration
storagedriver = [sd for sd in storagerouter.storagedrivers if sd.vpool_guid == vpool.guid][0]
storagedriver_config = StorageDriverConfiguration(
"storagedriver", vpool.guid, storagedriver.storagedriver_id
)
storagedriver_config.load()
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reconfigure)
tries = 5
while tries > 0:
try:
root_client = SSHClient(storagerouter, username="******")
root_client.dir_delete(directories=directories_to_clean, follow_symlinks=True)
for dir_name in directories_to_clean:
MDSServiceController._logger.debug("Recursively removed {0}".format(dir_name))
break
except Exception:
MDSServiceController._logger.debug("Waiting for the MDS service to go down...")
time.sleep(5)
tries -= 1
if tries == 0:
raise
except UnableToConnectException:
if allow_offline is True:
MDSServiceController._logger.info("Allowed offline node during mds service removal")
else:
raise
def prepare_mds_service(client, storagerouter, vpool, fresh_only=True, reload_config=False):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
"""
from ovs.lib.storagedriver import StorageDriverController
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid][0]
# Fetch service sequence number
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return None # There are already one or more MDS services running, aborting
service_number += 1
# Find free port
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.append(service.ports[0])
port = System.get_free_ports(Configuration.get('ovs.ports.mds'),
exclude=occupied_ports, nr=1, client=client)[0]
# Add service to the model
service = DalService()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name, service_number)
service.type = mdsservice_type
service.storagerouter = storagerouter
service.ports = [port]
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.save()
scrub_partition = None
db_partition = None
for disk in storagerouter.disks:
for partition in disk.partitions:
if DiskPartition.ROLES.DB in partition.roles:
db_partition = partition
if DiskPartition.ROLES.SCRUB in partition.roles:
scrub_partition = partition
if scrub_partition is None or db_partition is None:
raise RuntimeError('Could not find DB or SCRUB partition on StorageRouter {0}'.format(storagerouter.name))
StorageDriverController.add_storagedriverpartition(storagedriver, {'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': db_partition,
'mds_service': mds_service})
StorageDriverController.add_storagedriverpartition(storagedriver, {'size': None,
'role': DiskPartition.ROLES.SCRUB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': scrub_partition,
'mds_service': mds_service})
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': [sd_partition.path for sd_partition in mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.DB and sd_partition.sub_role == StorageDriverPartition.SUBROLE.MDS][0],
'scratch_directory': [sd_partition.path for sd_partition in mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.SCRUB and sd_partition.sub_role == StorageDriverPartition.SUBROLE.MDS][0]})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
return mds_service
def prepare_mds_service(client, storagerouter, vpool, fresh_only=True, reload_config=False):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
"""
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid][0]
# Fetch service sequence number
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return None # There are already one or more MDS services running, aborting
service_number += 1
# Find free port
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.append(service.ports[0])
port = System.get_free_ports(Configuration.get('ovs.ports.mds'),
exclude=occupied_ports, nr=1, client=client)[0]
# Add service to the model
service = DalService()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name, service_number)
service.type = mdsservice_type
service.storagerouter = storagerouter
service.ports = [port]
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.save()
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_md,
vpool.name,
mds_service.number),
'scratch_directory': '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_temp,
vpool.name,
mds_service.number)})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
return mds_service
def prepare_mds_service(client, storagerouter, vpool, fresh_only=True, start=False):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
"""
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid][0]
# Fetch service sequence number
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return None # There are already one or more MDS services running, aborting
service_number += 1
# Find free port
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.append(service.ports[0])
port = System.get_free_ports(Configuration.get('ovs.ports.mds'),
exclude=occupied_ports, nr=1, client=client)[0]
# Add service to the model
service = Service()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name, service_number)
service.type = mdsservice_type
service.storagerouter = storagerouter
service.ports = [port]
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.save()
# Prepare some directores
scratch_dir = '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_temp, vpool.name, service_number)
rocksdb_dir = '{0}/mds_{1}_{2}'.format(storagedriver.mountpoint_md, vpool.name, service_number)
client.run('mkdir -p {0}'.format(scratch_dir))
client.run('mkdir -p {0}'.format(rocksdb_dir))
# Generate the configuration file
metadataserver_config = StorageDriverConfiguration('metadataserver', vpool.name, number=service_number)
metadataserver_config.load(client)
metadataserver_config.clean() # Clean out obsolete values
if vpool.backend_type.code == 'alba':
metadataserver_config.configure_backend_connection_manager(alba_connection_host='127.0.0.1',
alba_connection_port=storagedriver.alba_proxy.service.ports[0],
backend_type='ALBA')
else:
metadataserver_config.configure_backend_connection_manager(**vpool.metadata)
metadataserver_config.configure_metadata_server(mds_address=storagerouter.ip,
mds_port=service.ports[0],
mds_scratch_dir=scratch_dir,
mds_rocksdb_path=rocksdb_dir)
metadataserver_config.save(client)
# Create system services
params = {'<VPOOL_NAME>': vpool.name,
'<SERVICE_NUMBER>': str(service_number)}
template_dir = '/opt/OpenvStorage/config/templates/upstart'
client.run('cp -f {0}/ovs-metadataserver.conf {0}/ovs-metadataserver_{1}_{2}.conf'.format(template_dir, vpool.name, service_number))
service_script = """
from ovs.plugin.provider.service import Service
Service.add_service(package=('openvstorage', 'metadataserver'), name='metadataserver_{0}_{1}', command=None, stop_command=None, params={2})
""".format(vpool.name, service_number, params)
System.exec_remote_python(client, service_script)
if start is True:
System.exec_remote_python(client, """
from ovs.plugin.provider.service import Service
Service.enable_service('{0}')
""".format(service.name))
System.exec_remote_python(client, """
from ovs.plugin.provider.service import Service
Service.start_service('{0}')
""".format(service.name))
return mds_service
def prepare_mds_service(cls, storagerouter, vpool):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and vPool are already configured with a StorageDriver and that all model-wise configurations regarding both have been completed.
:param storagerouter: StorageRouter on which the MDS service will be created
:type storagerouter: ovs.dal.hybrids.storagerouter.StorageRouter
:param vpool: The vPool for which the MDS service will be created
:type vpool: ovs.dal.hybrids.vpool.VPool
:raises RuntimeError: vPool is not extended on StorageRouter
No ServiceType found for 'MetadataServer'
No free port is found for the new MDSService
No partition found on StorageRouter with DB role
:return: Newly created junction service
:rtype: ovs.dal.hybrids.j_mdsservice.MDSService
"""
from ovs.lib.storagedriver import StorageDriverController # Import here to prevent from circular imports
cls._logger.info('StorageRouter {0} - vPool {1}: Preparing MDS junction service'.format(storagerouter.name, vpool.name))
mds_service = MDSService()
with volatile_mutex(name='prepare_mds_{0}'.format(storagerouter.guid), wait=30):
# VALIDATIONS
# Verify passed StorageRouter is part of the vPool
storagerouter.invalidate_dynamics(['vpools_guids'])
if vpool.guid not in storagerouter.vpools_guids:
raise RuntimeError('StorageRouter {0} is not part of vPool {1}'.format(storagerouter.name, vpool.name))
# Verify ServiceType existence
mds_service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.MD_SERVER)
if mds_service_type is None:
raise RuntimeError('No ServiceType found with name {0}'.format(ServiceType.SERVICE_TYPES.MD_SERVER))
# Retrieve occupied ports for current StorageRouter and max MDSService number for current vPool/StorageRouter combo
service_number = -1
occupied_ports = []
for service in mds_service_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.extend(service.ports)
if service.mds_service.vpool_guid == vpool.guid:
service_number = max(service.mds_service.number, service_number)
client = SSHClient(endpoint=storagerouter)
mds_port_range = Configuration.get('/ovs/framework/hosts/{0}/ports|mds'.format(System.get_my_machine_id(client)))
free_ports = System.get_free_ports(selected_range=mds_port_range,
exclude=occupied_ports,
amount=1,
client=client)
if len(free_ports) != 1:
raise RuntimeError('Failed to find an available port on StorageRouter {0} within range {1}'.format(storagerouter.name, mds_port_range))
# Partition check
db_partition = None
for disk in storagerouter.disks:
for partition in disk.partitions:
if DiskPartition.ROLES.DB in partition.roles:
db_partition = partition
break
if db_partition is None:
raise RuntimeError('Could not find DB partition on StorageRouter {0}'.format(storagerouter.name))
# Verify StorageDriver configured
storagedrivers = [sd for sd in vpool.storagedrivers if sd.storagerouter_guid == storagerouter.guid]
if len(storagedrivers) != 1:
raise RuntimeError('Expected to find a configured StorageDriver for vPool {0} on StorageRouter {1}'.format(vpool.name, storagerouter.name))
# MODEL UPDATES
# Service and MDS service
service_number += 1
cls._logger.info('StorageRouter {0} - vPool {1}: Adding junction service with number {2}'.format(storagerouter.name, vpool.name, service_number))
service = Service()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name, service_number)
service.type = mds_service_type
service.ports = free_ports
service.storagerouter = storagerouter
service.save()
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.service = service
mds_service.save()
# StorageDriver partitions
cls._logger.info('StorageRouter {0} - vPool {1}: Adding StorageDriverPartition on partition with mount point {2}'.format(storagerouter.name, vpool.name, db_partition.mountpoint))
storagedriver = storagedrivers[0]
sdp = StorageDriverController.add_storagedriverpartition(storagedriver, {'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': db_partition,
'mds_service': mds_service})
# CONFIGURATIONS
# Volumedriver
mds_nodes = []
for sd_partition in storagedriver.partitions:
if sd_partition.role == DiskPartition.ROLES.DB and sd_partition.sub_role == StorageDriverPartition.SUBROLE.MDS and sd_partition.mds_service is not None:
service = sd_partition.mds_service.service
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': '{0}/db'.format(sd_partition.path),
'scratch_directory': '{0}/scratch'.format(sd_partition.path)})
cls._logger.info('StorageRouter {0} - vPool {1}: Configuring StorageDriver with MDS nodes: {2}'.format(storagerouter.name, vpool.name, mds_nodes))
# Generate the correct section in the StorageDriver's configuration
try:
storagedriver_config = StorageDriverConfiguration(vpool.guid, storagedriver.storagedriver_id)
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client)
except Exception:
cls._logger.exception('StorageRouter {0} - vPool {1}: Configuring StorageDriver failed. Reverting model changes'.format(storagerouter.name, vpool.name))
# Clean up model changes if error occurs
sdp.delete()
mds_service.delete() # Must be removed before the service
service.delete()
return mds_service
