def create_hprm_config_files(vpool_guid, local_storagerouter_guid,
parameters):
"""
Create the required configuration files to be able to make use of HPRM (aka PRACC)
This configuration will be zipped and made available for download
:param vpool_guid: The guid of the VPool for which a HPRM manager needs to be deployed
:type vpool_guid: str
:param local_storagerouter_guid: The guid of the StorageRouter the API was requested on
:type local_storagerouter_guid: str
:param parameters: Additional information required for the HPRM configuration files
:type parameters: dict
:return: Name of the zipfile containing the configuration files
:rtype: str
"""
# Validations
required_params = {
'port': (int, {
'min': 1,
'max': 65535
}),
'identifier': (str, ExtensionsToolbox.regex_vpool)
}
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
vpool = VPool(vpool_guid)
identifier = parameters['identifier']
config_path = None
local_storagerouter = StorageRouter(local_storagerouter_guid)
for sd in vpool.storagedrivers:
if len(sd.alba_proxies) == 0:
raise ValueError(
'No ALBA proxies configured for vPool {0} on StorageRouter {1}'
.format(vpool.name, sd.storagerouter.name))
config_path = '/ovs/vpools/{0}/proxies/{1}/config/{{0}}'.format(
vpool.guid, sd.alba_proxies[0].guid)
if config_path is None:
raise ValueError(
'vPool {0} has not been extended any StorageRouter'.format(
vpool.name))
proxy_cfg = Configuration.get(key=config_path.format('main'))
cache_info = {}
arakoons = {}
cache_types = VPool.CACHES.values()
if not any(ctype in parameters for ctype in cache_types):
raise ValueError(
'At least one cache type should be passed: {0}'.format(
', '.join(cache_types)))
for ctype in cache_types:
if ctype not in parameters:
continue
required_dict = {'read': (bool, None), 'write': (bool, None)}
required_params.update({ctype: (dict, required_dict)})
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
read = parameters[ctype]['read']
write = parameters[ctype]['write']
if read is False and write is False:
cache_info[ctype] = ['none']
continue
path = parameters[ctype].get('path')
if path is not None:
path = path.strip()
if not path or path.endswith(
'/.') or '..' in path or '/./' in path:
raise ValueError('Invalid path specified')
required_dict.update({
'path': (str, None),
'size': (int, {
'min': 1,
'max': 10 * 1024
})
})
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
while '//' in path:
path = path.replace('//', '/')
cache_info[ctype] = [
'local', {
'path': path,
'max_size': parameters[ctype]['size'] * 1024**3,
'cache_on_read': read,
'cache_on_write': write
}
]
else:
required_dict.update({
'backend_info': (dict, {
'preset': (str, ExtensionsToolbox.regex_preset),
'alba_backend_guid':
(str, ExtensionsToolbox.regex_guid),
'alba_backend_name':
(str, ExtensionsToolbox.regex_backend)
}),
'connection_info': (dict, {
'host': (str, ExtensionsToolbox.regex_ip, False),
'port': (int, {
'min': 1,
'max': 65535
}, False),
'client_id':
(str, ExtensionsToolbox.regex_guid, False),
'client_secret': (str, None, False)
})
})
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
connection_info = parameters[ctype]['connection_info']
if connection_info[
'host']: # Remote Backend for accelerated Backend
alba_backend_guid = parameters[ctype]['backend_info'][
'alba_backend_guid']
ovs_client = OVSClient.get_instance(
connection_info=connection_info)
arakoon_config = VPoolShared.retrieve_alba_arakoon_config(
alba_backend_guid=alba_backend_guid,
ovs_client=ovs_client)
arakoons[ctype] = ArakoonClusterConfig.convert_config_to(
arakoon_config, return_type='INI')
else: # Local Backend for accelerated Backend
alba_backend_name = parameters[ctype]['backend_info'][
'alba_backend_name']
if Configuration.exists(key='/ovs/arakoon/{0}-abm/config'.
format(alba_backend_name),
raw=True) is False:
raise ValueError(
'Arakoon cluster for ALBA Backend {0} could not be retrieved'
.format(alba_backend_name))
arakoons[ctype] = Configuration.get(
key='/ovs/arakoon/{0}-abm/config'.format(
alba_backend_name),
raw=True)
cache_info[ctype] = [
'alba', {
'albamgr_cfg_url':
'/etc/hprm/{0}/{1}_cache_arakoon.ini'.format(
identifier, ctype),
'bucket_strategy': [
'1-to-1', {
'prefix':
vpool.guid,
'preset':
parameters[ctype]['backend_info']['preset']
}
],
'manifest_cache_size':
proxy_cfg['manifest_cache_size'],
'cache_on_read':
read,
'cache_on_write':
write
}
]
tgz_name = 'hprm_config_files_{0}_{1}.tgz'.format(
identifier, vpool.name)
config = {
'ips': ['127.0.0.1'],
'port': parameters['port'],
'pracc': {
'uds_path': '/var/run/hprm/{0}/uds_path'.format(identifier),
'max_clients': 1000,
'max_read_buf_size':
64 * 1024, # Buffer size for incoming requests (in bytes)
'thread_pool_size': 64
}, # Amount of threads
'transport': 'tcp',
'log_level': 'info',
'read_preference': proxy_cfg['read_preference'],
'albamgr_cfg_url': '/etc/hprm/{0}/arakoon.ini'.format(identifier),
'manifest_cache_size': proxy_cfg['manifest_cache_size']
}
file_contents_map = {}
for ctype in cache_types:
if ctype in cache_info:
config['{0}_cache'.format(ctype)] = cache_info[ctype]
if ctype in arakoons:
file_contents_map[
'/opt/OpenvStorage/config/{0}/{1}_cache_arakoon.ini'.
format(identifier, ctype)] = arakoons[ctype]
file_contents_map.update({
'/opt/OpenvStorage/config/{0}/config.json'.format(identifier):
json.dumps(config, indent=4),
'/opt/OpenvStorage/config/{0}/arakoon.ini'.format(identifier):
Configuration.get(key=config_path.format('abm'), raw=True)
})
local_client = SSHClient(endpoint=local_storagerouter)
local_client.dir_create(
directories='/opt/OpenvStorage/config/{0}'.format(identifier))
local_client.dir_create(
directories='/opt/OpenvStorage/webapps/frontend/downloads')
for file_name, contents in file_contents_map.iteritems():
local_client.file_write(contents=contents, filename=file_name)
local_client.run(command=[
'tar', '--transform', 's#^config/{0}#{0}#'.format(identifier),
'-czf', '/opt/OpenvStorage/webapps/frontend/downloads/{0}'.format(
tgz_name), 'config/{0}'.format(identifier)
])
local_client.dir_delete(
directories='/opt/OpenvStorage/config/{0}'.format(identifier))
return tgz_name
def setup_proxy_configs(self):
"""
Sets up the proxies their configuration data in the configuration management
:return: None
:rtype: NoneType
"""
def _generate_proxy_cache_config(cache_settings, cache_type, proxy_index):
if cache_settings['read'] is False and cache_settings['write'] is False:
return ['none']
if cache_settings['is_backend'] is True:
cfg_tree_name = 'abm_bc' if cache_type == StorageDriverConfiguration.CACHE_BLOCK else 'abm_aa'
return ['alba', {'cache_on_read': cache_settings['read'],
'cache_on_write': cache_settings['write'],
'albamgr_cfg_url': Configuration.get_configuration_path(config_tree.format(cfg_tree_name)),
'bucket_strategy': ['1-to-1', {'prefix': vpool.guid,
'preset': cache_settings['backend_info']['preset']}],
'manifest_cache_size': manifest_cache_size}]
if cache_type == StorageDriverConfiguration.CACHE_BLOCK:
path = '{0}/bc'.format(self.storagedriver_partitions_caches[proxy_index].path)
else:
path = '{0}/fc'.format(self.storagedriver_partitions_caches[proxy_index].path)
return ['local', {'path': path,
'max_size': self.cache_size_local / self.sr_installer.requested_local_proxies,
'cache_on_read': cache_settings['read'],
'cache_on_write': cache_settings['write']}]
def _generate_scrub_proxy_cache_config(cache_settings, main_proxy_cache_config):
scrub_cache_info = ['none']
if cache_settings['is_backend'] is True and cache_settings['write'] is True:
scrub_cache_info = copy.deepcopy(main_proxy_cache_config)
scrub_cache_info[1]['cache_on_read'] = False
return scrub_cache_info
def _generate_proxy_config(proxy_type, proxy_service):
proxy_config = {'log_level': 'info',
'port': proxy_service.service.ports[0] if proxy_type == 'main' else 0,
'ips': [self.storagedriver.storage_ip] if proxy_type == 'main' else ['127.0.0.1'],
'manifest_cache_size': manifest_cache_size,
'fragment_cache': fragment_cache_main_proxy if proxy_type == 'main' else fragment_cache_scrub_proxy,
'transport': 'tcp',
'read_preference': read_preferences,
'albamgr_cfg_url': Configuration.get_configuration_path(config_tree.format('abm'))}
if self.sr_installer.block_cache_supported:
proxy_config['block_cache'] = block_cache_main_proxy if proxy_type == 'main' else block_cache_scrub_proxy
return proxy_config
vpool = self.vp_installer.vpool
read_preferences = self.vp_installer.calculate_read_preferences()
manifest_cache_size = 500 * 1024 ** 2
block_cache_settings = vpool.metadata['caching_info'][self.storagedriver.storagerouter_guid][StorageDriverConfiguration.CACHE_BLOCK]
fragment_cache_settings = vpool.metadata['caching_info'][self.storagedriver.storagerouter_guid][StorageDriverConfiguration.CACHE_FRAGMENT]
# Obtain all arakoon configurations for each Backend (main, block cache, fragment cache)
arakoon_data = {'abm': vpool.metadata['backend']['backend_info']['arakoon_config']}
if block_cache_settings['is_backend'] is True:
arakoon_data['abm_bc'] = block_cache_settings['backend_info']['arakoon_config']
if fragment_cache_settings['is_backend'] is True:
arakoon_data['abm_aa'] = fragment_cache_settings['backend_info']['arakoon_config']
for proxy_id, alba_proxy in enumerate(self.storagedriver.alba_proxies):
config_tree = '/ovs/vpools/{0}/proxies/{1}/config/{{0}}'.format(vpool.guid, alba_proxy.guid)
for arakoon_entry, arakoon_config in arakoon_data.iteritems():
arakoon_config = ArakoonClusterConfig.convert_config_to(config=arakoon_config, return_type='INI')
Configuration.set(config_tree.format(arakoon_entry), arakoon_config, raw=True)
# Generate cache information for main proxy
block_cache_main_proxy = _generate_proxy_cache_config(cache_type=StorageDriverConfiguration.CACHE_BLOCK, cache_settings=block_cache_settings, proxy_index=proxy_id)
fragment_cache_main_proxy = _generate_proxy_cache_config(cache_type=StorageDriverConfiguration.CACHE_FRAGMENT, cache_settings=fragment_cache_settings, proxy_index=proxy_id)
# Generate cache information for scrub proxy
block_cache_scrub_proxy = _generate_scrub_proxy_cache_config(cache_settings=block_cache_settings, main_proxy_cache_config=block_cache_main_proxy)
fragment_cache_scrub_proxy = _generate_scrub_proxy_cache_config(cache_settings=fragment_cache_settings, main_proxy_cache_config=fragment_cache_main_proxy)
# Generate complete main and proxy configuration
main_proxy_config = _generate_proxy_config(proxy_type='main', proxy_service=alba_proxy)
scrub_proxy_config = _generate_proxy_config(proxy_type='scrub', proxy_service=alba_proxy)
# Add configurations to configuration management
Configuration.set(config_tree.format('main'), json.dumps(main_proxy_config, indent=4), raw=True)
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid), json.dumps(scrub_proxy_config, indent=4), raw=True)
