def index():
# type: () -> dict
"""
Retrieve the local node ID
:return: Node ID
:rtype: dict
"""
return SettingList.get_setting_by_code(code='node_id').value
def set_net(request_data):
# type: (dict) -> None
"""
Set IP information
:param request_data: Data about the request (given by the decorator)
:type request_data: dict
:return: None
:rtype: NoneType
"""
node_id = SettingList.get_setting_by_code(code='node_id').value
Configuration.set(
'{0}|ips'.format(ASD_NODE_CONFIG_NETWORK_LOCATION.format(node_id)),
request_data['ips'])
def uninitialize(cls):
# type: () -> List[str]
"""
Remove initially stored values from configuration store
:return: The currently registered users
:rtype: List[str]
"""
try:
node_id = SettingList.get_setting_by_code(code='node_id').value
except ObjectNotFoundException:
return
if cls.dir_exists(ASD_NODE_LOCATION.format(node_id)):
cls.delete(ASD_NODE_LOCATION.format(node_id))
return cls.unregister_usage(System.get_component_identifier())
def authorized(cls):
"""
Indicates whether a call is authenticated
"""
# For backwards compatibility we first try to retrieve the node ID by using the bootstrap file
try:
with open(BOOTSTRAP_FILE) as bstr_file:
node_id = json.load(bstr_file)['node_id']
except:
node_id = SettingList.get_setting_by_code(code='node_id').value
node_config = Configuration.get(
ASD_NODE_CONFIG_MAIN_LOCATION.format(node_id))
username = node_config['username']
password = node_config['password']
auth = request.authorization
return auth and auth.username == username and auth.password == password
def export(self):
"""
Exports this Disk's information to a dict structure
:return: Representation of the Disk as dict
:rtype: dict
"""
return {
'size': self.size,
'usage': self.usage,
'state': self.status['state'],
'device': '/dev/{0}'.format(self.name),
'aliases': self.aliases,
'node_id': SettingList.get_setting_by_code(code='node_id').value,
'available': self.available,
'mountpoint': self.mountpoint,
'state_detail': self.status.get('detail', ''),
'partition_amount': len(self.partitions),
'partition_aliases': self.partition_aliases
}
def sync_disks():
# type: () -> None
"""
Syncs the disks
Changes made to this code should be reflected in the framework DiskController.sync_with_reality call.
:return: None
:rtype: NoneType
"""
node_id = SettingList.get_setting_by_code(code='node_id').value
s3 = Configuration.get(
ASD_NODE_CONFIG_MAIN_LOCATION_S3.format(node_id), default=False)
disks, name_alias_mapping = DiskTools.model_devices(s3=s3)
disks_by_name = dict((disk.name, disk) for disk in disks)
alias_name_mapping = name_alias_mapping.reverse_mapping()
# Specific for the asd-manager: handle unique constraint exception
DiskController._prepare_for_name_switch(disks)
# Sync the model
for disk in DiskList.get_disks():
generic_disk_model = None # type: GenericDisk
for alias in disk.aliases:
# IBS wont have alias
if alias in alias_name_mapping:
name = alias_name_mapping[alias].replace('/dev/', '')
if name in disks_by_name:
generic_disk_model = disks_by_name.pop(name)
break
# Partitioned loop, nvme devices no longer show up in alias_name_mapping
if generic_disk_model is None and disk.name in disks_by_name and (
disk.name.startswith(tuple(['fio', 'loop', 'nvme']))):
generic_disk_model = disks_by_name.pop(disk.name)
if not generic_disk_model:
# Remove disk / partitions if not reported by 'lsblk'
DiskController._remove_disk_model(disk)
else:
# Update existing disks and their partitions
DiskController._sync_disk_with_model(disk, generic_disk_model)
# Create all disks and their partitions not yet modeled
for disk_name, generic_disk_model in disks_by_name.iteritems():
DiskController._model_disk(generic_disk_model)
def execute_migration_code(cls):
# type: () -> None
"""
Run some migration code after an update has been done
:return: None
:rtype: NoneType
"""
cls._logger.info('Starting out of band migrations for SDM nodes')
###########################
# Start crucial migration #
###########################
# Removal of bootstrap file and store API IP, API port and node ID in SQLite DB
try:
if cls._local_client.file_exists(BOOTSTRAP_FILE):
cls._logger.info('Bootstrap file still exists. Retrieving node ID')
with open(BOOTSTRAP_FILE) as bstr_file:
node_id = json.load(bstr_file)['node_id']
else:
node_id = SettingList.get_setting_by_code(code='node_id').value
except Exception:
cls._logger.exception('Unable to determine the node ID, cannot migrate')
raise
try:
api_settings_map = {'api_ip': 'ip', 'api_port': 'port'} # Map settings code to keys in the Config management
required_settings = ['node_id', 'migration_version'] + api_settings_map.keys()
for settings_code in required_settings:
try:
_ = SettingList.get_setting_by_code(settings_code)
except ObjectNotFoundException:
cls._logger.info('Missing required settings: {0}'.format(settings_code))
if settings_code == 'node_id':
value = node_id
elif settings_code in api_settings_map.keys():
# Information must be extracted from Configuration
main_config = Configuration.get(ASD_NODE_CONFIG_MAIN_LOCATION.format(node_id))
value = main_config[api_settings_map[settings_code]]
elif settings_code == 'migration_version':
# Introduce version for ASD Manager migration code
value = 0
else:
raise NotImplementedError('No action implemented for setting {0}'.format(settings_code))
cls._logger.info('Modeling Setting with code {0} and value {1}'.format(settings_code, value))
setting = Setting()
setting.code = settings_code
setting.value = value
setting.save()
if cls._local_client.file_exists(BOOTSTRAP_FILE):
cls._logger.info('Removing the bootstrap file')
cls._local_client.file_delete(BOOTSTRAP_FILE)
except Exception:
cls._logger.exception('Error during migration of code settings. Unable to proceed')
raise
###############################
# Start non-crucial migration #
###############################
errors = []
migration_setting = SettingList.get_setting_by_code(code='migration_version')
# Add installed package_name in version files and additional string replacements in service files
try:
if migration_setting.value < 1:
cls._logger.info('Adding additional information to service files')
edition = Configuration.get_edition()
if edition == PackageFactory.EDITION_ENTERPRISE:
for version_file_name in cls._local_client.file_list(directory=ServiceFactory.RUN_FILE_DIR):
version_file_path = '{0}/{1}'.format(ServiceFactory.RUN_FILE_DIR, version_file_name)
contents = cls._local_client.file_read(filename=version_file_path)
if '{0}='.format(PackageFactory.PKG_ALBA) in contents:
contents = contents.replace(PackageFactory.PKG_ALBA, PackageFactory.PKG_ALBA_EE)
cls._local_client.file_write(filename=version_file_path, contents=contents)
node_id = SettingList.get_setting_by_code(code='node_id').value
asd_services = list(ASDController.list_asd_services())
maint_services = list(MaintenanceController.get_services())
for service_name in asd_services + maint_services:
config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(node_id, service_name)
if Configuration.exists(key=config_key):
config = Configuration.get(key=config_key)
if 'RUN_FILE_DIR' in config:
continue
config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR
config['ALBA_PKG_NAME'] = PackageFactory.PKG_ALBA_EE
config['ALBA_VERSION_CMD'] = PackageFactory.VERSION_CMD_ALBA
Configuration.set(key=config_key, value=config)
cls._service_manager.regenerate_service(name=ASDController.ASD_PREFIX if service_name in asd_services else MaintenanceController.MAINTENANCE_PREFIX,
client=cls._local_client,
target_name=service_name)
except Exception as ex:
cls._logger.exception('Failed to regenerate the ASD and Maintenance services')
errors.append(ex)
try:
if migration_setting.value < 2:
if System.get_component_identifier() not in Configuration.get(Configuration.get_registration_key(), default=[]):
Configuration.register_usage(System.get_component_identifier())
except Exception as ex:
cls._logger.exception('Failed to register the asd-manager')
errors.append(ex)
if len(errors) == 0:
cls._logger.info('No errors during non-crucial migration. Saving the migration setting')
# Save migration settings when no errors occurred
migration_setting = SettingList.get_setting_by_code(code='migration_version')
migration_setting.value = 2
migration_setting.save()
cls._logger.info('Finished out of band migrations for SDM nodes')
def create_asd(disk):
"""
Creates and starts an ASD on a given disk
:param disk: Disk on which to create an ASD
:type disk: source.dal.objects.disk.Disk
:return: None
:rtype: NoneType
"""
# Validations
if disk.state == 'MISSING':
raise RuntimeError(
'Cannot create an ASD on missing disk {0}'.format(disk.name))
_node_id = SettingList.get_setting_by_code(code='node_id').value
ipaddresses = Configuration.get('{0}|ips'.format(
ASD_NODE_CONFIG_NETWORK_LOCATION.format(_node_id)))
if len(ipaddresses) == 0:
ipaddresses = OSFactory.get_manager().get_ip_addresses(
client=ASDController._local_client)
if len(ipaddresses) == 0:
raise RuntimeError('Could not find any IP on the local node')
alba_pkg_name, alba_version_cmd = PackageFactory.get_package_and_version_cmd_for(
component='alba'
) # Call here, because this potentially raises error, which should happen before actually making changes
# Fetch disk information
disk_size = int(
ASDController._local_client.run(
['df', '-B', '1', '--output=size', disk.mountpoint],
timeout=5).splitlines()[1])
# Find out appropriate disk size
asd_size = int(math.floor(disk_size / (len(disk.asds) + 1)))
for asd in disk.asds:
if asd.has_config:
config = Configuration.get(asd.config_key)
config['capacity'] = asd_size
cache_size = ASDController.calculate_rocksdb_cache_size(
is_ssd=disk.is_ssd)
if cache_size:
config.update({'rocksdb_block_cache_size': cache_size})
Configuration.set(asd.config_key, config)
try:
ASDController._service_manager.send_signal(
asd.service_name, signal.SIGUSR1,
ASDController._local_client)
except Exception as ex:
ASDController._logger.info(
'Could not send signal to ASD for reloading the quota: {0}'
.format(ex))
used_ports = []
for asd in ASDList.get_asds():
if asd.has_config:
config = Configuration.get(asd.config_key)
used_ports.append(config['port'])
if 'rora_port' in config:
used_ports.append(config['rora_port'])
# Prepare & start service
ASDController._logger.info('Setting up service for disk {0}'.format(
disk.name))
asd_id = ''.join(
random.choice(string.ascii_letters + string.digits)
for _ in range(32))
homedir = '{0}/{1}'.format(disk.mountpoint, asd_id)
base_port = Configuration.get('{0}|port'.format(
ASD_NODE_CONFIG_NETWORK_LOCATION.format(_node_id)))
asd_port = base_port
rora_port = base_port + 1
while asd_port in used_ports:
asd_port += 1
used_ports.append(asd_port)
while rora_port in used_ports:
rora_port += 1
asd_config = {
'ips': ipaddresses,
'home': homedir,
'port': asd_port,
'asd_id': asd_id,
'node_id': _node_id,
'capacity': asd_size,
'multicast': None,
'transport': 'tcp',
'log_level': 'info'
}
cache_size = ASDController.calculate_rocksdb_cache_size(
is_ssd=disk.is_ssd)
if cache_size:
asd_config.update({'rocksdb_block_cache_size': cache_size})
if Configuration.get('/ovs/framework/rdma'):
asd_config['rora_port'] = rora_port
asd_config['rora_transport'] = 'rdma'
if Configuration.exists('{0}/extra'.format(
ASD_NODE_CONFIG_LOCATION.format(_node_id))):
data = Configuration.get('{0}/extra'.format(
ASD_NODE_CONFIG_LOCATION.format(_node_id)))
asd_config.update(data)
asd = ASD()
asd.disk = disk
asd.port = asd_port
asd.hosts = ipaddresses
asd.asd_id = asd_id
asd.folder = asd_id
asd.save()
Configuration.set(asd.config_key, asd_config)
params = {
'LOG_SINK': Logger.get_sink_path('alba-asd_{0}'.format(asd_id)),
'CONFIG_PATH':
Configuration.get_configuration_path(asd.config_key),
'SERVICE_NAME': asd.service_name,
'ALBA_PKG_NAME': alba_pkg_name,
'ALBA_VERSION_CMD': alba_version_cmd
}
os.mkdir(homedir)
ASDController._local_client.run(['chown', '-R', 'alba:alba', homedir])
ASDController._service_manager.add_service(
name=ASDController.ASD_PREFIX,
client=ASDController._local_client,
params=params,
target_name=asd.service_name)
ASDController.start_asd(asd)
"""
Print the message and log it using the logger instance
"""
getattr(asd_manager_logger, level)(message, extra={'print_msg': True})
if __name__ == '__main__':
def _sync_disks():
from source.controllers.disk import DiskController
while True:
DiskController.sync_disks()
time.sleep(60)
try:
node_id = SettingList.get_setting_by_code(code='node_id').value
except:
# For backwards compatibility
# After update SettingList has not been populated yet and post-update script of package will restart asd-manager
with open(BOOTSTRAP_FILE) as bstr_file:
node_id = json.load(bstr_file)['node_id']
try:
asd_manager_config = Configuration.get(
ASD_NODE_CONFIG_MAIN_LOCATION.format(node_id))
except:
raise RuntimeError('Configuration management unavailable')
if 'ip' not in asd_manager_config or 'port' not in asd_manager_config:
raise RuntimeError(
'IP and/or port not available in configuration for ALBA node {0}'.
format(node_id))