def _map_to_user_inputs(self, hadoop_version, configs):
config_objs = self.get_all_configs(hadoop_version)
# convert config objects to applicable_target -> config_name -> obj
config_objs_map = {}
for config_obj in config_objs:
applicable_target = config_obj.applicable_target
confs = config_objs_map.get(applicable_target, {})
confs[config_obj.name] = config_obj
config_objs_map[applicable_target] = confs
# iterate over all configs and append UserInputs to result list
result = []
for applicable_target in configs:
for config_name in configs[applicable_target]:
confs = config_objs_map.get(applicable_target)
if not confs:
raise ex.ConfigurationError(
_("Can't find applicable target "
"'%(applicable_target)s' for '%(config_name)s'") % {
"applicable_target": applicable_target,
"config_name": config_name
})
conf = confs.get(config_name)
if not conf:
raise ex.ConfigurationError(
_("Can't find config '%(config_name)s' "
"in '%(applicable_target)s'") % {
"config_name": config_name,
"applicable_target": applicable_target
})
result.append(
UserInput(conf, configs[applicable_target][config_name]))
return sorted(result)
def _load_job_binaries(self):
config_jb = CONF.job_binary_types
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_jb,
namespace='sahara.job_binary.types',
invoke_on_load=True
)
for ext in extension_manager.extensions:
if ext.name in self.job_binaries:
raise ex.ConfigurationError(
_("Job binary with name '%s' already exists.") %
ext.name)
ext.obj.name = ext.name
self.job_binaries[ext.name] = ext.obj
LOG.info(_LI("Job binary name {jb_name} loaded {entry_point}")
.format(jb_name=ext.name,
entry_point=ext.entry_point_target))
if len(self.job_binaries) < len(config_jb):
loaded_jb = set(six.iterkeys(self.job_binaries))
requested_jb = set(config_jb)
raise ex.ConfigurationError(
_("Job binaries couldn't be loaded: %s") %
", ".join(requested_jb - loaded_jb))
def domain_for_proxy():
'''Return the proxy domain or None
If configured to use the proxy domain, this function will return that
domain. If not configured to use the proxy domain, this function will
return None. If the proxy domain can't be found this will raise an
exception.
:returns: A Keystone Domain object or None.
:raises ConfigurationError: If the domain is requested but not specified.
:raises NotFoundException: If the domain name is specified but cannot be
found.
'''
if CONF.use_domain_for_proxy_users is False:
return None
if CONF.proxy_user_domain_name is None:
raise ex.ConfigurationError(_('Proxy domain requested but not '
'specified.'))
admin = k.client_for_admin()
global PROXY_DOMAIN
if not PROXY_DOMAIN:
domain_list = admin.domains.list(name=CONF.proxy_user_domain_name)
if len(domain_list) == 0:
raise ex.NotFoundException(value=CONF.proxy_user_domain_name,
message=_('Failed to find domain %s'))
# the domain name should be globally unique in Keystone
if len(domain_list) > 1:
raise ex.NotFoundException(value=CONF.proxy_user_domain_name,
message=_('Unexpected results found '
'when searching for domain '
'%s'))
PROXY_DOMAIN = domain_list[0]
return PROXY_DOMAIN
def parse_configs(conf_files=None):
try:
version_string = version.version_info.version_string()
CONF(project='sahara', version=version_string,
default_config_files=conf_files)
except cfg.RequiredOptError as roe:
raise ex.ConfigurationError(
_("Option '%(option)s' is required for config group '%(group)s'") %
{'option': roe.opt_name, 'group': roe.group.name})
def get_config_value(service, name, cluster=None):
if cluster:
for ng in cluster.node_groups:
if (ng.configuration().get(service) and
ng.configuration()[service].get(name)):
return ng.configuration()[service][name]
raise ex.ConfigurationError(
_("Unable to get parameter '%(param_name)s' from "
"service %(service)s") % {'param_name': name, 'service': service})
def parse_configs(conf_files=None):
try:
version_string = version.version_info.version_string()
CONF(project='sahara',
version=version_string,
default_config_files=conf_files)
except cfg.RequiredOptError as roe:
raise ex.ConfigurationError(
"Option '%s' is required for config group '%s'" %
(roe.opt_name, roe.group.name))
validate_configs()
def _load_data_sources(self):
config_ds = CONF.data_source_types
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_ds,
namespace='sahara.data_source.types',
invoke_on_load=True)
for ext in extension_manager.extensions:
if ext.name in self.data_sources:
raise ex.ConfigurationError(
_("Data source with name '%s' already exists.") % ext.name)
ext.obj.name = ext.name
self.data_sources[ext.name] = ext.obj
LOG.info("Data source name {ds_name} loaded {entry_point}".format(
ds_name=ext.name, entry_point=ext.entry_point_target))
if len(self.data_sources) < len(config_ds):
loaded_ds = set(six.iterkeys(self.data_sources))
requested_ds = set(config_ds)
raise ex.ConfigurationError(
_("Data sources couldn't be loaded: %s") %
", ".join(requested_ds - loaded_ds))
def _load_cluster_plugins(self):
config_plugins = CONF.plugins
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_plugins,
namespace='sahara.cluster.plugins',
invoke_on_load=True)
for ext in extension_manager.extensions:
if ext.name in self.plugins:
raise ex.ConfigurationError(
"Plugin with name '%s' already exists." % ext.name)
ext.obj.name = ext.name
self.plugins[ext.name] = ext.obj
LOG.info("Plugin '%s' loaded (%s)" %
(ext.name, ext.entry_point_target))
if len(self.plugins) < len(config_plugins):
loaded_plugins = set(six.iterkeys(self.plugins))
requested_plugins = set(config_plugins)
raise ex.ConfigurationError(
"Plugins couldn't be loaded: %s" %
", ".join(requested_plugins - loaded_plugins))
def get_config_value(service, name, cluster=None):
if cluster:
sahara_configs = generate_sahara_configs(cluster)
if sahara_configs.get(name):
return sahara_configs[name]
for ng in cluster.node_groups:
if (ng.configuration().get(service)
and ng.configuration()[service].get(name)):
return ng.configuration()[service][name]
for c in PLUGIN_CONFIGS:
if c.applicable_target == service and c.name == name:
return c.default_value
raise ex.ConfigurationError("Unable get parameter '%s' from service %s" %
(name, service))
def get_config_value(service, name, cluster=None):
if cluster:
sahara_configs = generate_sahara_configs(cluster)
if sahara_configs.get(name):
return sahara_configs[name]
for ng in cluster.node_groups:
if (ng.configuration().get(service)
and ng.configuration()[service].get(name)):
return ng.configuration()[service][name]
for configs in PLUGIN_CONFIGS:
if configs.applicable_target == service and configs.name == name:
return configs.default_value
raise ex.ConfigurationError(
_("Unable get parameter '%(parameter)s' from "
"service %(service)s") % {
"parameter": name,
"service": service
})
def validate_network_configs():
if CONF.use_namespaces and not CONF.use_neutron:
raise ex.ConfigurationError(
_('use_namespaces can not be set to "True" when use_neutron '
'is set to "False"'))